diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/checkpoints/checkpoint-100000.pth b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/checkpoints/checkpoint-100000.pth new file mode 100644 index 0000000000000000000000000000000000000000..e97f8792823ba3d11e3a3c1a21117c6391f58537 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/checkpoints/checkpoint-100000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96c8e1aa40a53f4b33cae9dafddd169a555c15fc75f987a9fdf13a61b46e4934 +size 149229259 diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_10000.json b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..3708bd925b45db404654d5bb46f572f960d03c46 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 0.043092675107085296, "train": {"Classification_binary": 3.866671562194824, "Classification": 0.01132840197533369, "Regression_pt": 0.2771618962287903, "Regression_eta": 0.07497438043355942, "Regression_sin_phi": 0.05248326435685158, "Regression_cos_phi": 0.05254027992486954, "Regression_energy": 0.33735641837120056, "Total": 12.252510070800781, "Total_unweighted": 4.672516203485429}, "train_group": {"physical_pu": {"Classification_binary": 3.106679292825552, "Classification": 0.015894242490713414, "Regression_pt": 0.2514718296108665, "Regression_eta": 0.10841400356410624, "Regression_sin_phi": 0.04795049568945235, "Regression_cos_phi": 0.04801140459520476, "Regression_energy": 0.29774360800837424, "Total": 11.271631052205851, "Total_unweighted": 3.87616487678427}, "physical_nopu": {"Classification_binary": 3.822851923563824, "Classification": 0.015336702056270097, "Regression_pt": 0.43421984887687026, "Regression_eta": 0.07304071311247755, "Regression_sin_phi": 0.049359811176889146, "Regression_cos_phi": 0.049595242200216445, "Regression_energy": 0.48183886816661614, "Total": 13.882375276521488, "Total_unweighted": 4.926243109153163}}, "valid": {"Classification_binary": 3.443409283955892, "Classification": 0.0156454232831796, "Regression_pt": 0.34263583024342853, "Regression_eta": 0.09105993310610454, "Regression_sin_phi": 0.04897646605968475, "Regression_cos_phi": 0.04892761011918386, "Regression_energy": 0.3879047632217407, "Total": 12.564829508463541, "group/physical_pu/Classification": 0.0159162903825442, "group/physical_pu/Classification_binary": 3.0796194076538086, "group/physical_pu/Regression_cos_phi": 0.04830873509248098, "group/physical_pu/Regression_energy": 0.29635826746622723, "group/physical_pu/Regression_eta": 0.10787934064865112, "group/physical_pu/Regression_pt": 0.25087740023930866, "group/physical_pu/Regression_sin_phi": 0.04817114273707072, "group/physical_pu/Total": 11.246471405029297, "group/physical_pu/Total_unweighted": 3.8471305842200914, "group/physical_nopu/Classification": 0.015374556183815002, "group/physical_nopu/Classification_binary": 3.807199160257975, "group/physical_nopu/Regression_cos_phi": 0.04954648514588674, "group/physical_nopu/Regression_energy": 0.4794512589772542, "group/physical_nopu/Regression_eta": 0.0742405354976654, "group/physical_nopu/Regression_pt": 0.4343942006429036, "group/physical_nopu/Regression_sin_phi": 0.049781784415245056, "group/physical_nopu/Total": 13.883186340332031, "group/physical_nopu/Total_unweighted": 4.909987981120745, "Total_unweighted": 4.378559309989214, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.015443235635757446, "diagnostic/loss/cms_pf/Classification_binary": 3.4404847621917725, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.048034630715847015, "diagnostic/loss/cms_pf/Regression_energy": 0.3758031129837036, "diagnostic/loss/cms_pf/Regression_eta": 0.09109895676374435, "diagnostic/loss/cms_pf/Regression_pt": 0.3299483358860016, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.04714006930589676, "diagnostic/regression/cms_pf/energy_pred_mean": 0.22193695604801178, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.3726426959037781, "diagnostic/regression/cms_pf/energy_residual_mean": 0.11117072403430939, "diagnostic/regression/cms_pf/energy_residual_rms": 0.6012683510780334, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.13191407918930054, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.3441822826862335, "diagnostic/regression/cms_pf/pt_residual_mean": 0.14268626272678375, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5678381323814392, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8830735683441162, "metrics/particle/class_1/pid_accuracy": 0.9767484068870544, "metrics/particle/class_2/efficiency": 0.2595495879650116, "metrics/particle/class_2/pid_accuracy": 0.19067919254302979, "metrics/particle/class_3/efficiency": 0.36635324358940125, "metrics/particle/class_3/pid_accuracy": 0.4670664072036743, "metrics/particle/class_4/efficiency": 0.4197738766670227, "metrics/particle/class_4/pid_accuracy": 0.6342803835868835, "metrics/particle/class_5/efficiency": 0.35690879821777344, "metrics/particle/class_5/pid_accuracy": 0.5831088423728943, "metrics/particle/class_6/efficiency": 0.8689244985580444, "metrics/particle/class_6/pid_accuracy": 0.0326157845556736, "metrics/particle/class_7/efficiency": 0.962668776512146, "metrics/particle/class_7/pid_accuracy": 0.25, "metrics/particle/count/bias_mean": 63.72037887573242, "metrics/particle/count/mae": 87.99967193603516, "metrics/particle/count/prediction_mean": 1592.3238525390625, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.1058100163936615, "metrics/particle/event/energy_response_mean": 0.9187098145484924, "metrics/particle/event/met_abs_error": 30.682153701782227, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06300193816423416, "metrics/particle/event/scalar_pt_response_mean": 1.0229040384292603, "metrics/particle/event/vector_pt_closure": 0.03837931528687477, "metrics/particle/matched/delta_eta_abs_mean": 0.022808415815234184, "metrics/particle/matched/delta_phi_abs_mean": 0.022895053029060364, "metrics/particle/matched/delta_r_mean": 0.035845596343278885, "metrics/particle/matched/energy_relative_abs_mean": 0.1227416843175888, "metrics/particle/matched/pid_accuracy": 0.8409348130226135, "metrics/particle/matched/pt_relative_abs_mean": 0.10608841478824615, "metrics/particle/matching/duplicate_fraction": 0.10800698399543762, "metrics/particle/matching/efficiency": 0.6241225004196167, "metrics/particle/matching/f1": 0.6113796830177307, "metrics/particle/matching/purity": 0.599146842956543, "metrics/particle/matching/target_coverage_dr0p05": 0.4967595040798187, "metrics/particle/matching/target_coverage_dr0p10": 0.691890299320221, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5486429333686829, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7429206967353821, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02762877568602562, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07643187046051025, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05826551839709282, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.007607629522681236, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1080533042550087, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0019488607067614794, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.2295636683702469, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.011454046703875065, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4619795083999634, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68245.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4739377796649933, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0900235176086426, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.3972230553627014, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.2820156216621399, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14086852967739105, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.2121224403381348, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.038999564945697784, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8038055896759033, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.12082187086343765, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 8.580384254455566, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87642.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.778599202632904, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 108.41983795166016, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 53.536033630371094, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 20.08069610595703, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 2.0091352462768555, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -2.3987016677856445, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 46.572654724121094, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -5.177511215209961, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 72.03817749023438, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04555178061127663, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16241860389709473, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15954774618148804, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.011004379950463772, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2127087563276291, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17014379799365997, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.294193267822266, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.11552706360816956, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 10.973824501037598, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 117530.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.601058006286621, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9968996047973633, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.0288289785385132, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.8166677951812744, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.33597591519355774, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 5.104506969451904, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.3888949751853943, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 16.727710723876953, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.5721912384033203, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 34.813724517822266, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228311.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5081037878990173, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -201.61256408691406, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 693.0384521484375, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 23.302227020263672, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 1.7422758340835571, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -4.53267240524292, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 691.2659301757812, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 553.8535766601562, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 1025.89111328125, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.026113316416740417, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09715788066387177, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.05499815195798874, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02016126550734043, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1086236834526062, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006995751173235476, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.09426810592412949, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.00195969152264297, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0134977102279663, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 58607.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3531205654144287, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.4062436819076538, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.2990415394306183, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.21043141186237335, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11801370978355408, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 3.0187880992889404, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.02378281019628048, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0940396785736084, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.06452164053916931, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.207996845245361, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72195.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8117538094520569, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 35.46520233154297, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 32.966392517089844, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 25.157867431640625, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 4.76000452041626, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -6.691399574279785, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 18.640153884887695, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.806931495666504, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 46.01987838745117, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.035885006189346313, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12275569885969162, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10619133710861206, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.0016671818448230624, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16688214242458344, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08253881335258484, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.6328368186950684, "metrics/particle/momentum/all/accepted/pz_bias": 0.05922870710492134, "metrics/particle/momentum/all/accepted/pz_residual_rms": 7.66542911529541, "metrics/particle/momentum/all/accepted/samples": 244382.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.1144449710845947, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0940639972686768, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7504758834838867, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.583186686038971, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.25138092041015625, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 4.2574920654296875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.24197986721992493, "metrics/particle/momentum/all/assigned/pz_abs_mean": 10.450121879577637, "metrics/particle/momentum/all/assigned/pz_bias": 0.29728439450263977, "metrics/particle/momentum/all/assigned/pz_residual_rms": 27.14212417602539, "metrics/particle/momentum/all/assigned/samples": 388148.0, "metrics/particle/momentum/all/efficiency": 0.624762237071991, "metrics/particle/momentum/all/event/energy_bias": -57.727516174316406, "metrics/particle/momentum/all/event/p3_error_mean": 688.6898803710938, "metrics/particle/momentum/all/event/pt_vector_error_mean": 40.20012283325195, "metrics/particle/momentum/all/event/px_bias": 8.5114164352417, "metrics/particle/momentum/all/event/py_bias": -13.622775077819824, "metrics/particle/momentum/all/event/pz_abs_mean": 682.6458129882812, "metrics/particle/momentum/all/event/pz_bias": 546.869140625, "metrics/particle/momentum/all/event/pz_residual_rms": 1012.8251953125, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.025832023471593857, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.07593940943479538, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05420154705643654, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.015426421537995338, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09622964262962341, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012915042461827397, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.21761207282543182, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.00540861627086997, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.224713683128357, "metrics/particle/momentum/class_1/accepted/samples": 170344.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.245698943734169, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.9073656797409058, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3418518304824829, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.28979364037513733, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08286304026842117, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.531435489654541, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.02424943447113037, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.184385061264038, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0014893916668370366, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.27687406539917, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8835819363594055, "metrics/particle/momentum/class_1/event/energy_bias": 87.0224609375, "metrics/particle/momentum/class_1/event/p3_error_mean": 66.7913818359375, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 24.52997589111328, "metrics/particle/momentum/class_1/event/px_bias": -3.1484742164611816, "metrics/particle/momentum/class_1/event/py_bias": -7.799188613891602, "metrics/particle/momentum/class_1/event/pz_abs_mean": 57.41948699951172, "metrics/particle/momentum/class_1/event/pz_bias": 3.8108339309692383, "metrics/particle/momentum/class_1/event/pz_residual_rms": 86.51609802246094, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.058834366500377655, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23846901953220367, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23177100718021393, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07081083208322525, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27155590057373047, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.10328638553619385, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 3.9673049449920654, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.101127989590168, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 8.701680183410645, "metrics/particle/momentum/class_2/accepted/samples": 8094.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.1733858585357666, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.4562780857086182, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1697120666503906, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6907708644866943, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.47481968998908997, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.650973796844482, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.35149407386779785, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.476667404174805, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.08435860276222229, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 34.799129486083984, "metrics/particle/momentum/class_2/assigned/samples": 31056.0, "metrics/particle/momentum/class_2/efficiency": 0.2605420649051666, "metrics/particle/momentum/class_2/event/energy_bias": -429.0236511230469, "metrics/particle/momentum/class_2/event/p3_error_mean": 198.000244140625, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 31.710887908935547, "metrics/particle/momentum/class_2/event/px_bias": 8.89044189453125, "metrics/particle/momentum/class_2/event/py_bias": -5.761624813079834, "metrics/particle/momentum/class_2/event/pz_abs_mean": 190.91268920898438, "metrics/particle/momentum/class_2/event/pz_bias": -105.81779479980469, "metrics/particle/momentum/class_2/event/pz_residual_rms": 279.791259765625, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.06128362566232681, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.2266392707824707, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.22399869561195374, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.005362731870263815, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2637207508087158, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.3150818347930908, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.17325210571289, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.14614734053611755, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 13.363001823425293, "metrics/particle/momentum/class_3/accepted/samples": 19487.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.2061116695404053, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.0140392780303955, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.1864676475524902, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.013351559638977, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.44682514667510986, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 3.720524549484253, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5592039823532104, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 21.98485565185547, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.6810662150382996, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 36.45990753173828, "metrics/particle/momentum/class_3/assigned/samples": 52412.0, "metrics/particle/momentum/class_3/efficiency": 0.36560291051864624, "metrics/particle/momentum/class_3/event/energy_bias": 569.3394165039062, "metrics/particle/momentum/class_3/event/p3_error_mean": 289.6946105957031, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.657726287841797, "metrics/particle/momentum/class_3/event/px_bias": -2.1980209350585938, "metrics/particle/momentum/class_3/event/py_bias": -2.678313732147217, "metrics/particle/momentum/class_3/event/pz_abs_mean": 289.18341064453125, "metrics/particle/momentum/class_3/event/pz_bias": 4.515160083770752, "metrics/particle/momentum/class_3/event/pz_residual_rms": 450.91021728515625, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.06027613580226898, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.233389213681221, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.23031052947044373, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.05452854186296463, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2692371606826782, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.3586931824684143, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 10.772046089172363, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2875223159790039, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 16.59093475341797, "metrics/particle/momentum/class_4/accepted/samples": 35384.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.9519485235214233, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.1134307384490967, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.1665805578231812, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8942509293556213, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.388500839471817, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 4.907932758331299, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5439158082008362, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 24.0845947265625, "metrics/particle/momentum/class_4/assigned/pz_bias": 1.0700758695602417, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 44.145015716552734, "metrics/particle/momentum/class_4/assigned/samples": 82180.0, "metrics/particle/momentum/class_4/efficiency": 0.4198186993598938, "metrics/particle/momentum/class_4/event/energy_bias": -323.11871337890625, "metrics/particle/momentum/class_4/event/p3_error_mean": 736.2532958984375, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 13.549942970275879, "metrics/particle/momentum/class_4/event/px_bias": -0.16025780141353607, "metrics/particle/momentum/class_4/event/py_bias": 4.179305553436279, "metrics/particle/momentum/class_4/event/pz_abs_mean": 735.6431274414062, "metrics/particle/momentum/class_4/event/pz_bias": 654.2223510742188, "metrics/particle/momentum/class_4/event/pz_residual_rms": 1066.4306640625, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.052833616733551025, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.23076027631759644, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21911416947841644, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03849261254072189, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2598878741264343, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.026407092809677124, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.3281508684158325, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.04699701815843582, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.019072532653809, "metrics/particle/momentum/class_5/accepted/samples": 10376.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4331071376800537, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.0544180870056152, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 1.061366081237793, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.7681432366371155, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.39563894271850586, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 5.67451810836792, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.14855898916721344, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.33903694152832, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.20882770419120789, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 16.904319763183594, "metrics/particle/momentum/class_5/assigned/samples": 28938.0, "metrics/particle/momentum/class_5/efficiency": 0.35844820737838745, "metrics/particle/momentum/class_5/event/energy_bias": 49.21307373046875, "metrics/particle/momentum/class_5/event/p3_error_mean": 124.99046325683594, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 27.243297576904297, "metrics/particle/momentum/class_5/event/px_bias": 4.598775863647461, "metrics/particle/momentum/class_5/event/py_bias": -1.3003811836242676, "metrics/particle/momentum/class_5/event/pz_abs_mean": 117.23223114013672, "metrics/particle/momentum/class_5/event/pz_bias": -11.052337646484375, "metrics/particle/momentum/class_5/event/pz_residual_rms": 179.34144592285156, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.026952816173434258, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.10291888564825058, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.09205103665590286, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.03572550415992737, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.13806864619255066, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.011928429827094078, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.835676372051239, "metrics/particle/momentum/class_6/accepted/pz_bias": 0.41933882236480713, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 4.3792338371276855, "metrics/particle/momentum/class_6/accepted/samples": 503.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.32111793756484985, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 0.673272430896759, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.4961214065551758, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.3558274209499359, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.09565217047929764, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 2.2153072357177734, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.03999999910593033, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 1.8311808109283447, "metrics/particle/momentum/class_6/assigned/pz_bias": 0.24583320319652557, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 7.959249019622803, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8747826218605042, "metrics/particle/momentum/class_6/event/energy_bias": -8.771795272827148, "metrics/particle/momentum/class_6/event/p3_error_mean": 8.280181884765625, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 3.2573952674865723, "metrics/particle/momentum/class_6/event/px_bias": -0.29936787486076355, "metrics/particle/momentum/class_6/event/py_bias": -0.4834509491920471, "metrics/particle/momentum/class_6/event/pz_abs_mean": 7.156983375549316, "metrics/particle/momentum/class_6/event/pz_bias": -0.704095721244812, "metrics/particle/momentum/class_6/event/pz_residual_rms": 19.340333938598633, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.022219790145754814, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.06387217342853546, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.04220566153526306, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.027245627716183662, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.07307195663452148, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.015463917516171932, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.6141220927238464, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.05347319692373276, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 1.8060885667800903, "metrics/particle/momentum/class_7/accepted/samples": 194.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.05707685649394989, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.07981337606906891, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.05214940011501312, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.03509155660867691, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.01005025114864111, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.11210285127162933, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.015075377188622952, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.6277675032615662, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.05088556557893753, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 1.7938120365142822, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9748743772506714, "metrics/particle/momentum/class_7/event/energy_bias": -2.388338088989258, "metrics/particle/momentum/class_7/event/p3_error_mean": 4.4943318367004395, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 1.9264230728149414, "metrics/particle/momentum/class_7/event/px_bias": 0.828319251537323, "metrics/particle/momentum/class_7/event/py_bias": 0.22087903320789337, "metrics/particle/momentum/class_7/event/pz_abs_mean": 3.457505702972412, "metrics/particle/momentum/class_7/event/pz_bias": 1.8950684070587158, "metrics/particle/momentum/class_7/event/pz_residual_rms": 21.221227645874023, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 3.8684284687042236, "step_valid_time": 1923.804875612259, "step_total_time": 1927.6733045578003, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0839745784528028, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.41661100166668563, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 86799, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15219, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11833, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18679349493709727, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.1753361213838869, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.1808835589574147, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8246638786161131, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14523473458115987, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.13632645537390983, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14063967101275301, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18679349493709727, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0723722889965481, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.4349951905103556, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 86707, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 14288, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11055, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.17587611861298144, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.16478485012744068, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.1701499291438915, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8352151498725593, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1360799615948005, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.12749835653407451, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.13164945875459969, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.17587611861298144, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0758677540148975, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.44406424338002015, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 86083, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 13722, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 10580, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.1698498557972001, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.15940429585400137, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.16446138357543508, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8405957041459986, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1309584225575264, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.12290463854651906, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.12680377774581716, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.1698498557972001} \ No newline at end of file diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_100000.json b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_100000.json new file mode 100644 index 0000000000000000000000000000000000000000..87c64c3ce941595fc728f57fbd8d4ea7367557a6 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_100000.json @@ -0,0 +1 @@ +{"epoch": 0.430926751070853, "train": {"Classification_binary": 2.913294792175293, "Classification": 0.01590229757130146, "Regression_pt": 0.2420644611120224, "Regression_eta": 0.10410107672214508, "Regression_sin_phi": 0.04839209094643593, "Regression_cos_phi": 0.047524116933345795, "Regression_energy": 0.28326350450515747, "Total": 10.890850067138672, "Total_unweighted": 3.654542339965701}, "train_group": {"physical_pu": {"Classification_binary": 2.91530542583256, "Classification": 0.015567265164393645, "Regression_pt": 0.22943693432179127, "Regression_eta": 0.10514523743928134, "Regression_sin_phi": 0.047267391704596005, "Regression_cos_phi": 0.047256517909712845, "Regression_energy": 0.2736412899179773, "Total": 10.712314385634203, "Total_unweighted": 3.633620062290312}, "physical_nopu": {"Classification_binary": 3.464650022445863, "Classification": 0.014358685933856747, "Regression_pt": 0.37127626565161054, "Regression_eta": 0.06930408466144232, "Regression_sin_phi": 0.04739802543020616, "Regression_cos_phi": 0.04748998504906598, "Regression_energy": 0.4166025387571983, "Total": 12.553274857591349, "Total_unweighted": 4.431079607929243}}, "valid": {"Classification_binary": 3.1898606618245444, "Classification": 0.01496494064728419, "Regression_pt": 0.30063915252685547, "Regression_eta": 0.08770813544591267, "Regression_sin_phi": 0.04753067592779795, "Regression_cos_phi": 0.04750901460647583, "Regression_energy": 0.34531140327453613, "Total": 11.652684529622396, "group/physical_pu/Classification": 0.015510041266679764, "group/physical_pu/Classification_binary": 2.913379987080892, "group/physical_pu/Regression_cos_phi": 0.04732653498649597, "group/physical_pu/Regression_energy": 0.27315084139506024, "group/physical_pu/Regression_eta": 0.10474092761675517, "group/physical_pu/Regression_pt": 0.22885777552922568, "group/physical_pu/Regression_sin_phi": 0.04728392759958903, "group/physical_pu/Total": 10.701224009195963, "group/physical_pu/Total_unweighted": 3.630250035474698, "group/physical_nopu/Classification": 0.014419838786125183, "group/physical_nopu/Classification_binary": 3.466341018676758, "group/physical_nopu/Regression_cos_phi": 0.04769149422645569, "group/physical_nopu/Regression_energy": 0.4174719254175822, "group/physical_nopu/Regression_eta": 0.07067534327507019, "group/physical_nopu/Regression_pt": 0.37242058912913006, "group/physical_nopu/Regression_sin_phi": 0.047777424256006874, "group/physical_nopu/Total": 12.604145050048828, "group/physical_nopu/Total_unweighted": 4.436797633767128, "Total_unweighted": 4.0335239842534065, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.01474827527999878, "diagnostic/loss/cms_pf/Classification_binary": 3.184847593307495, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.046954698860645294, "diagnostic/loss/cms_pf/Regression_energy": 0.332868367433548, "diagnostic/loss/cms_pf/Regression_eta": 0.08788224309682846, "diagnostic/loss/cms_pf/Regression_pt": 0.2882271409034729, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.04566651210188866, "diagnostic/regression/cms_pf/energy_pred_mean": 0.21647006273269653, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.3550470471382141, "diagnostic/regression/cms_pf/energy_residual_mean": 0.10570383816957474, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5831822752952576, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.11252348870038986, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.32661306858062744, "diagnostic/regression/cms_pf/pt_residual_mean": 0.12329566478729248, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5437797904014587, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8925094604492188, "metrics/particle/class_1/pid_accuracy": 0.9783657789230347, "metrics/particle/class_2/efficiency": 0.2581618130207062, "metrics/particle/class_2/pid_accuracy": 0.20871298015117645, "metrics/particle/class_3/efficiency": 0.39595770835876465, "metrics/particle/class_3/pid_accuracy": 0.47804272174835205, "metrics/particle/class_4/efficiency": 0.4413287341594696, "metrics/particle/class_4/pid_accuracy": 0.6602673530578613, "metrics/particle/class_5/efficiency": 0.38023120164871216, "metrics/particle/class_5/pid_accuracy": 0.6368086338043213, "metrics/particle/class_6/efficiency": 0.8632563352584839, "metrics/particle/class_6/pid_accuracy": 0.055318448692560196, "metrics/particle/class_7/efficiency": 0.9650516510009766, "metrics/particle/class_7/pid_accuracy": 0.24979424476623535, "metrics/particle/count/bias_mean": 70.52603912353516, "metrics/particle/count/mae": 91.810546875, "metrics/particle/count/prediction_mean": 1599.1295166015625, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.08270182460546494, "metrics/particle/event/energy_response_mean": 0.9407674670219421, "metrics/particle/event/met_abs_error": 18.390907287597656, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04612429067492485, "metrics/particle/event/scalar_pt_response_mean": 1.0064588785171509, "metrics/particle/event/vector_pt_closure": 0.02754901349544525, "metrics/particle/matched/delta_eta_abs_mean": 0.018709123134613037, "metrics/particle/matched/delta_phi_abs_mean": 0.01602235808968544, "metrics/particle/matched/delta_r_mean": 0.02761870250105858, "metrics/particle/matched/energy_relative_abs_mean": 0.11835885047912598, "metrics/particle/matched/pid_accuracy": 0.8463555574417114, "metrics/particle/matched/pt_relative_abs_mean": 0.10528282076120377, "metrics/particle/matching/duplicate_fraction": 0.11146851629018784, "metrics/particle/matching/efficiency": 0.6390694379806519, "metrics/particle/matching/f1": 0.6246592998504639, "metrics/particle/matching/purity": 0.6108847260475159, "metrics/particle/matching/target_coverage_dr0p05": 0.5157459378242493, "metrics/particle/matching/target_coverage_dr0p10": 0.7034327983856201, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5584445595741272, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7564361691474915, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01713113859295845, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06950735300779343, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05832326412200928, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0012704514665529132, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1102716401219368, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.001963293645530939, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.2246875762939453, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 1.6203559425775893e-05, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.422878384590149, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68762.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.46526390314102173, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0725276470184326, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.36686521768569946, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.23930901288986206, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13624590635299683, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.1339101791381836, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03869105502963066, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.797707438468933, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0713273137807846, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.215193748474121, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87643.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7844976186752319, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 57.5850715637207, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 42.9327278137207, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 15.10973834991455, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.19163842499256134, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.11095701903104782, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 38.30548095703125, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -4.395179748535156, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 58.35177993774414, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04041844233870506, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16165593266487122, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15895257890224457, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.014745497144758701, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21237602829933167, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.1736265867948532, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.496962547302246, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.007971935905516148, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.643301963806152, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 121577.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.521260380744934, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9335994124412537, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9330704808235168, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7183117866516113, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3195055425167084, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3.5537519454956055, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.3772245943546295, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.68471908569336, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.0752953290939331, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.179996490478516, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228694.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5255997180938721, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 129.34866333007812, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 376.62896728515625, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 19.150972366333008, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.11161485314369202, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.9572997689247131, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 374.9935607910156, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -31.89948272705078, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 550.7647094726562, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013411147519946098, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08673529326915741, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.050143592059612274, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008538561873137951, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10696781426668167, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005686189397238195, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.07647530734539032, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.00686780596151948, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7415221929550171, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59794.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3338879644870758, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.5736192464828491, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.2679661512374878, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.17075105011463165, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10399179905653, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.138452053070068, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.022161278873682022, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0696443319320679, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.06438272446393967, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.656369209289551, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72198.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8281946778297424, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 17.581357955932617, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 19.93624496459961, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 14.683895111083984, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.9524204730987549, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.6604888439178467, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 11.239407539367676, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.056751132011413574, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 21.418611526489258, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.027560662478208542, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11841442435979843, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10527875274419785, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.005475156009197235, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16733364760875702, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08506674319505692, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.7518434524536133, "metrics/particle/momentum/all/accepted/pz_bias": 0.002237466862425208, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.15965747833252, "metrics/particle/momentum/all/accepted/samples": 250133.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0624172687530518, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.083867073059082, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6817594170570374, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5085133910179138, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.23812012374401093, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.9322521686553955, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.23488231003284454, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.836394309997559, "metrics/particle/momentum/all/assigned/pz_bias": -0.07237245887517929, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.237295150756836, "metrics/particle/momentum/all/assigned/samples": 388535.0, "metrics/particle/momentum/all/efficiency": 0.6394646763801575, "metrics/particle/momentum/all/event/energy_bias": 204.5150909423828, "metrics/particle/momentum/all/event/p3_error_mean": 377.52685546875, "metrics/particle/momentum/all/event/pt_vector_error_mean": 25.249971389770508, "metrics/particle/momentum/all/event/px_bias": 1.8723969459533691, "metrics/particle/momentum/all/event/py_bias": -2.728745460510254, "metrics/particle/momentum/all/event/pz_abs_mean": 374.7854309082031, "metrics/particle/momentum/all/event/pz_bias": -36.35141372680664, "metrics/particle/momentum/all/event/pz_residual_rms": 548.2797241210938, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.013742339797317982, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06536377221345901, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05174361914396286, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005914163310080767, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09611216932535172, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011844350956380367, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.20280900597572327, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.00454637361690402, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.0820857286453247, "metrics/particle/momentum/class_1/accepted/samples": 172234.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.21938158571720123, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.924846351146698, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3030761182308197, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24382668733596802, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07418511807918549, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.351126670837402, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.02228354476392269, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1090704202651978, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.016589386388659477, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.464953422546387, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8933854699134827, "metrics/particle/momentum/class_1/event/energy_bias": 51.29511260986328, "metrics/particle/momentum/class_1/event/p3_error_mean": 55.0443115234375, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 16.264562606811523, "metrics/particle/momentum/class_1/event/px_bias": -0.5810542106628418, "metrics/particle/momentum/class_1/event/py_bias": -1.3434793949127197, "metrics/particle/momentum/class_1/event/pz_abs_mean": 49.58595275878906, "metrics/particle/momentum/class_1/event/pz_bias": -3.79465389251709, "metrics/particle/momentum/class_1/event/pz_residual_rms": 80.329345703125, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.058353226631879807, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2366930991411209, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23295047879219055, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06996621936559677, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2731923460960388, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.10996648669242859, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.1626763343811035, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.16163212060928345, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 9.648889541625977, "metrics/particle/momentum/class_2/accepted/samples": 8057.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.195249080657959, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.4924668073654175, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1112648248672485, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.618732750415802, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4743350148200989, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.920191764831543, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.3512590825557709, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.586763381958008, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.014554296620190144, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 35.16551971435547, "metrics/particle/momentum/class_2/assigned/samples": 31054.0, "metrics/particle/momentum/class_2/efficiency": 0.25935104489326477, "metrics/particle/momentum/class_2/event/energy_bias": -364.5095520019531, "metrics/particle/momentum/class_2/event/p3_error_mean": 170.8944549560547, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 28.27145004272461, "metrics/particle/momentum/class_2/event/px_bias": 1.8596668243408203, "metrics/particle/momentum/class_2/event/py_bias": -3.5815203189849854, "metrics/particle/momentum/class_2/event/pz_abs_mean": 165.0124969482422, "metrics/particle/momentum/class_2/event/pz_bias": -20.952621459960938, "metrics/particle/momentum/class_2/event/pz_residual_rms": 231.46434020996094, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.06061739847064018, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.2277713268995285, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.22502164542675018, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.005490296520292759, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2643662989139557, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.31611961126327515, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.373311042785645, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.10258965194225311, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 13.960383415222168, "metrics/particle/momentum/class_3/accepted/samples": 21204.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.0296223163604736, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.9471911191940308, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.0903747081756592, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.9131590127944946, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.42159217596054077, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 2.74595308303833, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5358599424362183, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.30038070678711, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.0628027617931366, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.424171447753906, "metrics/particle/momentum/class_3/assigned/samples": 52482.0, "metrics/particle/momentum/class_3/efficiency": 0.39781618118286133, "metrics/particle/momentum/class_3/event/energy_bias": 632.7193603515625, "metrics/particle/momentum/class_3/event/p3_error_mean": 283.6934509277344, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.467957496643066, "metrics/particle/momentum/class_3/event/px_bias": -0.3674440383911133, "metrics/particle/momentum/class_3/event/py_bias": -0.6369193196296692, "metrics/particle/momentum/class_3/event/pz_abs_mean": 283.25067138671875, "metrics/particle/momentum/class_3/event/pz_bias": 30.980289459228516, "metrics/particle/momentum/class_3/event/pz_residual_rms": 448.56494140625, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.059614408761262894, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.23034179210662842, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22691355645656586, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.05671189725399017, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.26613980531692505, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.3582344353199005, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 11.011812210083008, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.06493481993675232, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.45818328857422, "metrics/particle/momentum/class_4/accepted/samples": 36906.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.8673808574676514, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.049911379814148, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0564379692077637, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.7855201363563538, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3668331205844879, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.173112392425537, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5286552309989929, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.333703994750977, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.1907520741224289, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 39.85127258300781, "metrics/particle/momentum/class_4/assigned/samples": 82498.0, "metrics/particle/momentum/class_4/efficiency": 0.43787670135498047, "metrics/particle/momentum/class_4/event/energy_bias": -108.13228607177734, "metrics/particle/momentum/class_4/event/p3_error_mean": 322.24786376953125, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 10.85108757019043, "metrics/particle/momentum/class_4/event/px_bias": -0.22293661534786224, "metrics/particle/momentum/class_4/event/py_bias": 0.4545741677284241, "metrics/particle/momentum/class_4/event/pz_abs_mean": 321.693359375, "metrics/particle/momentum/class_4/event/pz_bias": -36.27690124511719, "metrics/particle/momentum/class_4/event/pz_residual_rms": 481.0196228027344, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.05065150931477547, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.27760985493659973, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21231263875961304, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03644867241382599, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2526898682117462, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.022908367216587067, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.281274437904358, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.021333269774913788, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.304769039154053, "metrics/particle/momentum/class_5/accepted/samples": 11044.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4360547065734863, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.0649726390838623, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9414138197898865, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6354988217353821, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.38128477334976196, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.604482650756836, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.1484847366809845, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.354648113250732, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.18817853927612305, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 17.285776138305664, "metrics/particle/momentum/class_5/assigned/samples": 28939.0, "metrics/particle/momentum/class_5/efficiency": 0.38152486085891724, "metrics/particle/momentum/class_5/event/energy_bias": 4.1957502365112305, "metrics/particle/momentum/class_5/event/p3_error_mean": 113.18727111816406, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 25.459196090698242, "metrics/particle/momentum/class_5/event/px_bias": 1.831383466720581, "metrics/particle/momentum/class_5/event/py_bias": 3.1269702911376953, "metrics/particle/momentum/class_5/event/pz_abs_mean": 105.48094177246094, "metrics/particle/momentum/class_5/event/pz_bias": -5.535974979400635, "metrics/particle/momentum/class_5/event/pz_residual_rms": 166.05201721191406, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.019038870930671692, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.09465982764959335, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.09255222231149673, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.04075591266155243, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.13539637625217438, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.016194332391023636, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.8488697409629822, "metrics/particle/momentum/class_6/accepted/pz_bias": 0.1546376347541809, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 4.277896881103516, "metrics/particle/momentum/class_6/accepted/samples": 494.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.30932554602622986, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 0.6396656632423401, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.5485613346099854, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.4029461145401001, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.104347825050354, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 2.5438477993011475, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.03478260710835457, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 1.6470446586608887, "metrics/particle/momentum/class_6/assigned/pz_bias": 0.031112849712371826, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 6.489809036254883, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8591304421424866, "metrics/particle/momentum/class_6/event/energy_bias": -9.361015319824219, "metrics/particle/momentum/class_6/event/p3_error_mean": 7.621616363525391, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 3.0306735038757324, "metrics/particle/momentum/class_6/event/px_bias": -0.5495069622993469, "metrics/particle/momentum/class_6/event/py_bias": -0.6446943879127502, "metrics/particle/momentum/class_6/event/pz_abs_mean": 6.54725456237793, "metrics/particle/momentum/class_6/event/pz_bias": -0.6174635887145996, "metrics/particle/momentum/class_6/event/pz_residual_rms": 18.42405891418457, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.012984729371964931, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.05729352682828903, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.0436273068189621, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.01340466644614935, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.08095896244049072, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.015463917516171932, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.6920047402381897, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.11095403879880905, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 2.527327060699463, "metrics/particle/momentum/class_7/accepted/samples": 194.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.05626257136464119, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.10971520841121674, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.053750623017549515, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.018153145909309387, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.015075377188622952, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.12073169648647308, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.015075377188622952, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.7449876666069031, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.042916666716337204, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 2.5706627368927, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9748743772506714, "metrics/particle/momentum/class_7/event/energy_bias": -1.6922376155853271, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.2944552898406982, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 1.0017496347427368, "metrics/particle/momentum/class_7/event/px_bias": -0.0977112278342247, "metrics/particle/momentum/class_7/event/py_bias": -0.10367649048566818, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.8965392112731934, "metrics/particle/momentum/class_7/event/pz_bias": -0.15408790111541748, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.92692756652832, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 4.4297401905059814, "step_valid_time": 20910.223651647568, "step_total_time": 20914.65339231491, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0485542792708873, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3552566040873216, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84411, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16872, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13686, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.20708192697146363, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.1998791626683726, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.20341680431139458, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8001208373316274, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1679779073335379, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.16213526673063938, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.1650048828713695, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.20708192697146363, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0489525567089522, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.37771686450677544, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84104, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16002, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12776, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19697435960560813, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19026443450965472, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19356126355515504, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8097355654903453, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15726436809906572, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15190716256063921, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15453935152985007, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19697435960560813, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0509526029967229, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3845496034242465, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 83652, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15346, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12220, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18995160232209832, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18345048534404437, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18664444998510105, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8165495146559556, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15125821584621668, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14608138478458374, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14862473470728102, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18995160232209832} \ No newline at end of file diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_20000.json b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..b7cf3f667237fe1ad2404d479848731780f44341 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 0.08618535021417059, "train": {"Classification_binary": 3.7799065113067627, "Classification": 0.012396327219903469, "Regression_pt": 0.37137407064437866, "Regression_eta": 0.06541791558265686, "Regression_sin_phi": 0.051129456609487534, "Regression_cos_phi": 0.05173248425126076, "Regression_energy": 0.42513519525527954, "Total": 13.104599952697754, "Total_unweighted": 4.7570919608697295}, "train_group": {"physical_pu": {"Classification_binary": 3.036711187153072, "Classification": 0.015791614642264425, "Regression_pt": 0.2412811414553569, "Regression_eta": 0.10691552248957394, "Regression_sin_phi": 0.04779683340054292, "Regression_cos_phi": 0.047741924218096574, "Regression_energy": 0.2872921899779812, "Total": 11.044519288199288, "Total_unweighted": 3.783530413336888}, "physical_nopu": {"Classification_binary": 3.678874814995576, "Classification": 0.015008531790534066, "Regression_pt": 0.4058778911697851, "Regression_eta": 0.07119528032896673, "Regression_sin_phi": 0.048263121699572135, "Regression_cos_phi": 0.04861910777664421, "Regression_energy": 0.4527475386366199, "Total": 13.294988318507297, "Total_unweighted": 4.720586286397698}}, "valid": {"Classification_binary": 3.3462375005086265, "Classification": 0.015462325265010199, "Regression_pt": 0.32527021567026776, "Regression_eta": 0.08971320589383443, "Regression_sin_phi": 0.04841866592566172, "Regression_cos_phi": 0.04834861556688944, "Regression_energy": 0.37056442101796466, "Total": 12.200915018717447, "group/physical_pu/Classification": 0.015821824471155804, "group/physical_pu/Classification_binary": 3.022279739379883, "group/physical_pu/Regression_cos_phi": 0.04785729944705963, "group/physical_pu/Regression_energy": 0.2870272994041443, "group/physical_pu/Regression_eta": 0.10661615928014119, "group/physical_pu/Regression_pt": 0.24146680037180582, "group/physical_pu/Regression_sin_phi": 0.047817895809809365, "group/physical_pu/Total": 11.032371520996094, "group/physical_pu/Total_unweighted": 3.768887018163999, "group/physical_nopu/Classification": 0.015102823575337728, "group/physical_nopu/Classification_binary": 3.6701955795288086, "group/physical_nopu/Regression_cos_phi": 0.04883993665377299, "group/physical_nopu/Regression_energy": 0.4541015625, "group/physical_nopu/Regression_eta": 0.07281026244163513, "group/physical_nopu/Regression_pt": 0.4090735912322998, "group/physical_nopu/Regression_sin_phi": 0.049019436041514076, "group/physical_nopu/Total": 13.369458516438803, "group/physical_nopu/Total_unweighted": 4.719143191973369, "Total_unweighted": 4.244014949848254, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.01522118877619505, "diagnostic/loss/cms_pf/Classification_binary": 3.339203357696533, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.04763038083910942, "diagnostic/loss/cms_pf/Regression_energy": 0.3559514582157135, "diagnostic/loss/cms_pf/Regression_eta": 0.08991075307130814, "diagnostic/loss/cms_pf/Regression_pt": 0.31008070707321167, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.04664941877126694, "diagnostic/regression/cms_pf/energy_pred_mean": 0.22051919996738434, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.3633172810077667, "diagnostic/regression/cms_pf/energy_residual_mean": 0.10975298285484314, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5972575545310974, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.13049906492233276, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.33792462944984436, "diagnostic/regression/cms_pf/pt_residual_mean": 0.14127123355865479, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5607213973999023, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8866716623306274, "metrics/particle/class_1/pid_accuracy": 0.977197527885437, "metrics/particle/class_2/efficiency": 0.25407087802886963, "metrics/particle/class_2/pid_accuracy": 0.20475563406944275, "metrics/particle/class_3/efficiency": 0.3700796663761139, "metrics/particle/class_3/pid_accuracy": 0.4742179214954376, "metrics/particle/class_4/efficiency": 0.422744482755661, "metrics/particle/class_4/pid_accuracy": 0.6434751152992249, "metrics/particle/class_5/efficiency": 0.35981348156929016, "metrics/particle/class_5/pid_accuracy": 0.6021636128425598, "metrics/particle/class_6/efficiency": 0.8673657178878784, "metrics/particle/class_6/pid_accuracy": 0.03953602537512779, "metrics/particle/class_7/efficiency": 0.9622716307640076, "metrics/particle/class_7/pid_accuracy": 0.21914981305599213, "metrics/particle/count/bias_mean": 63.02180862426758, "metrics/particle/count/mae": 85.8642578125, "metrics/particle/count/prediction_mean": 1591.6253662109375, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.11158271878957748, "metrics/particle/event/energy_response_mean": 0.9408469796180725, "metrics/particle/event/met_abs_error": 25.69482421875, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05612586438655853, "metrics/particle/event/scalar_pt_response_mean": 1.0101712942123413, "metrics/particle/event/vector_pt_closure": 0.0335441492497921, "metrics/particle/matched/delta_eta_abs_mean": 0.021716710180044174, "metrics/particle/matched/delta_phi_abs_mean": 0.018285872414708138, "metrics/particle/matched/delta_r_mean": 0.03158013895153999, "metrics/particle/matched/energy_relative_abs_mean": 0.11370950937271118, "metrics/particle/matched/pid_accuracy": 0.8446420431137085, "metrics/particle/matched/pt_relative_abs_mean": 0.10440296679735184, "metrics/particle/matching/duplicate_fraction": 0.11264955997467041, "metrics/particle/matching/efficiency": 0.6268203854560852, "metrics/particle/matching/f1": 0.6141600012779236, "metrics/particle/matching/purity": 0.602000892162323, "metrics/particle/matching/target_coverage_dr0p05": 0.5078328251838684, "metrics/particle/matching/target_coverage_dr0p10": 0.6938788890838623, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5528002381324768, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7452629804611206, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02188345044851303, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06599808484315872, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05738614872097969, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0007003101636655629, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.10715997219085693, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0018321998650208116, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.21294595301151276, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.014153999276459217, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.3486618995666504, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68224.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4744754433631897, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0792797803878784, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.37791287899017334, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.25031623244285583, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.140511617064476, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.2171595096588135, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.039512116461992264, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8256038427352905, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.15995647013187408, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.05367660522461, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87644.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7783596515655518, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 53.56079864501953, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 48.37178039550781, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 18.69717025756836, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.1399613618850708, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.4149754047393799, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 42.03236389160156, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.22188717126846313, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 66.18016815185547, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.042984481900930405, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15969425439834595, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15808865427970886, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.009981232695281506, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21145498752593994, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.1754140555858612, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.501962184906006, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.07328057289123535, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.500478744506836, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 117921.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5881502628326416, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.006301760673523, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9999512434005737, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7939318418502808, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3344925343990326, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3.99790620803833, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.390055388212204, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 16.36762237548828, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.06794673949480057, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 33.44773864746094, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228606.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5097941756248474, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 794.2716064453125, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 399.1051330566406, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 21.88561248779297, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 4.151066303253174, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -3.107950448989868, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 396.8376159667969, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 34.7552604675293, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 592.3134765625, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.019879575818777084, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.07736068218946457, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.05183281749486923, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013474867679178715, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10576105862855911, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006932350224815309, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.08325760066509247, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0172865130007267, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8569644689559937, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59143.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.34093186259269714, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3072956800460815, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.27340441942214966, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.17658351361751556, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1104462742805481, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 2.7929704189300537, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.021981218829751015, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0492712259292603, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.08668752014636993, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.414792060852051, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72198.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8191778063774109, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 9.009879112243652, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 29.01800537109375, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 21.39246368408203, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.438633441925049, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -4.8221940994262695, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 17.19605255126953, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.3142081499099731, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 43.737403869628906, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03154449909925461, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11378176510334015, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10445937514305115, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.0013546316185966134, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16548891365528107, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08500619977712631, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.724344491958618, "metrics/particle/momentum/all/accepted/pz_bias": 0.02712443470954895, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.016658782958984, "metrics/particle/momentum/all/accepted/samples": 245288.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.1050649881362915, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0787110328674316, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7245652079582214, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5565358996391296, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2490835338830948, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.4590911865234375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.24255241453647614, "metrics/particle/momentum/all/assigned/pz_abs_mean": 10.239452362060547, "metrics/particle/momentum/all/assigned/pz_bias": -0.012214911170303822, "metrics/particle/momentum/all/assigned/pz_residual_rms": 26.16371726989746, "metrics/particle/momentum/all/assigned/samples": 388448.0, "metrics/particle/momentum/all/efficiency": 0.6270784139633179, "metrics/particle/momentum/all/event/energy_bias": 856.8423461914062, "metrics/particle/momentum/all/event/p3_error_mean": 403.2171325683594, "metrics/particle/momentum/all/event/pt_vector_error_mean": 33.82640838623047, "metrics/particle/momentum/all/event/px_bias": 1.5724709033966064, "metrics/particle/momentum/all/event/py_bias": -8.34511947631836, "metrics/particle/momentum/all/event/pz_abs_mean": 396.7284240722656, "metrics/particle/momentum/all/event/pz_bias": 33.662933349609375, "metrics/particle/momentum/all/event/pz_residual_rms": 590.1044921875, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01965080387890339, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06410650163888931, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05252528563141823, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.010156853124499321, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09485545009374619, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012628993717953563, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.2089429795742035, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.020806901156902313, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1660394668579102, "metrics/particle/momentum/class_1/accepted/samples": 171035.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.23086383938789368, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.8673127293586731, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3175729513168335, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2608315646648407, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07900387793779373, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.039229154586792, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.022693321108818054, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1522284746170044, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02147510088980198, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.454313278198242, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8871662020683289, "metrics/particle/momentum/class_1/event/energy_bias": 45.215904235839844, "metrics/particle/momentum/class_1/event/p3_error_mean": 62.659725189208984, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 21.693132400512695, "metrics/particle/momentum/class_1/event/px_bias": -4.9626336097717285, "metrics/particle/momentum/class_1/event/py_bias": -2.874786138534546, "metrics/particle/momentum/class_1/event/pz_abs_mean": 54.7025146484375, "metrics/particle/momentum/class_1/event/pz_bias": -20.6525821685791, "metrics/particle/momentum/class_1/event/pz_residual_rms": 87.3509750366211, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0587187185883522, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.24033725261688232, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23316575586795807, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07852782309055328, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27310988306999207, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.11012934148311615, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.0782670974731445, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.0984230488538742, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 8.781900405883789, "metrics/particle/momentum/class_2/accepted/samples": 7809.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.203880548477173, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.463810682296753, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.16605806350708, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6742679476737976, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.47644805908203125, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.656472682952881, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.35229724645614624, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.74585723876953, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.3229069411754608, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 35.53069305419922, "metrics/particle/momentum/class_2/assigned/samples": 31059.0, "metrics/particle/momentum/class_2/efficiency": 0.25136804580688477, "metrics/particle/momentum/class_2/event/energy_bias": -330.5858154296875, "metrics/particle/momentum/class_2/event/p3_error_mean": 179.19851684570312, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 30.17165184020996, "metrics/particle/momentum/class_2/event/px_bias": 0.9189341068267822, "metrics/particle/momentum/class_2/event/py_bias": -7.3082098960876465, "metrics/particle/momentum/class_2/event/pz_abs_mean": 172.24884033203125, "metrics/particle/momentum/class_2/event/pz_bias": -62.797786712646484, "metrics/particle/momentum/class_2/event/pz_residual_rms": 247.94932556152344, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.060951463878154755, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.2264195829629898, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.22365204989910126, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.001740188105031848, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2632754445075989, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.329624205827713, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.629600524902344, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.19590777158737183, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 14.119997024536133, "metrics/particle/momentum/class_3/accepted/samples": 19771.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.163447141647339, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.0527839660644531, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.164602279663086, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.0025278329849243, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.44920051097869873, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 2.836481809616089, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5634731650352478, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 21.500858306884766, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.03243248909711838, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 34.93553161621094, "metrics/particle/momentum/class_3/assigned/samples": 52471.0, "metrics/particle/momentum/class_3/efficiency": 0.37093111872673035, "metrics/particle/momentum/class_3/event/energy_bias": 1004.467041015625, "metrics/particle/momentum/class_3/event/p3_error_mean": 336.57806396484375, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.213532447814941, "metrics/particle/momentum/class_3/event/px_bias": -0.6160758137702942, "metrics/particle/momentum/class_3/event/py_bias": -2.749459743499756, "metrics/particle/momentum/class_3/event/pz_abs_mean": 336.0652770996094, "metrics/particle/momentum/class_3/event/pz_bias": -199.4037322998047, "metrics/particle/momentum/class_3/event/pz_residual_rms": 537.188232421875, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.06022874638438225, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.2299804985523224, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.2277022898197174, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.04812520369887352, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.26678165793418884, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.36625513434410095, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 11.17595100402832, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.17317944765090942, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.42523765563965, "metrics/particle/momentum/class_4/accepted/samples": 35478.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.9455997943878174, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.131704568862915, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.140268325805664, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8827312588691711, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3863103985786438, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.9870214462280273, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5467705130577087, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 23.269346237182617, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2281949520111084, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 41.61384582519531, "metrics/particle/momentum/class_4/assigned/samples": 82413.0, "metrics/particle/momentum/class_4/efficiency": 0.4209339916706085, "metrics/particle/momentum/class_4/event/energy_bias": 192.4574432373047, "metrics/particle/momentum/class_4/event/p3_error_mean": 461.50433349609375, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 13.267008781433105, "metrics/particle/momentum/class_4/event/px_bias": 6.7049784660339355, "metrics/particle/momentum/class_4/event/py_bias": 3.7333388328552246, "metrics/particle/momentum/class_4/event/pz_abs_mean": 460.8904724121094, "metrics/particle/momentum/class_4/event/pz_bias": 320.76513671875, "metrics/particle/momentum/class_4/event/pz_residual_rms": 668.3006591796875, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.05346075817942619, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.226042240858078, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2153082937002182, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02230052463710308, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.25626635551452637, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.024213537573814392, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2583588361740112, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.05768904089927673, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.420891761779785, "metrics/particle/momentum/class_5/accepted/samples": 10490.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.460418462753296, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 1.9761981964111328, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9879149794578552, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6688646674156189, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.389351487159729, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.9689548015594482, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.14663304388523102, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.491819381713867, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.25818932056427, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 17.797210693359375, "metrics/particle/momentum/class_5/assigned/samples": 28943.0, "metrics/particle/momentum/class_5/efficiency": 0.36238643527030945, "metrics/particle/momentum/class_5/event/energy_bias": -44.635719299316406, "metrics/particle/momentum/class_5/event/p3_error_mean": 111.91490173339844, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 25.49875259399414, "metrics/particle/momentum/class_5/event/px_bias": -0.33216118812561035, "metrics/particle/momentum/class_5/event/py_bias": 1.334187626838684, "metrics/particle/momentum/class_5/event/pz_abs_mean": 103.87728881835938, "metrics/particle/momentum/class_5/event/pz_bias": -3.2332868576049805, "metrics/particle/momentum/class_5/event/pz_residual_rms": 160.9818572998047, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.023243533447384834, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.09765411168336868, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.09879736602306366, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.04099640995264053, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.14389421045780182, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.017543859779834747, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.8469361662864685, "metrics/particle/momentum/class_6/accepted/pz_bias": 0.20185989141464233, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 4.117358684539795, "metrics/particle/momentum/class_6/accepted/samples": 513.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.2840542197227478, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 1.0867977142333984, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.5662159323692322, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.4133880138397217, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.07999999821186066, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 3.0980491638183594, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.05043478310108185, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 2.30128812789917, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.216313898563385, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 10.315780639648438, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8921738862991333, "metrics/particle/momentum/class_6/event/energy_bias": -8.968106269836426, "metrics/particle/momentum/class_6/event/p3_error_mean": 8.68252944946289, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 3.298978805541992, "metrics/particle/momentum/class_6/event/px_bias": -0.3148675560951233, "metrics/particle/momentum/class_6/event/py_bias": -0.45760804414749146, "metrics/particle/momentum/class_6/event/pz_abs_mean": 7.520907878875732, "metrics/particle/momentum/class_6/event/pz_bias": -1.0099635124206543, "metrics/particle/momentum/class_6/event/pz_residual_rms": 21.202547073364258, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.01762164756655693, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.05721443518996239, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.04512737691402435, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.020057303830981255, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.08063307404518127, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.0052083334885537624, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.7469647526741028, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.3767014443874359, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 2.328442335128784, "metrics/particle/momentum/class_7/accepted/samples": 192.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.04513855651021004, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.10740978270769119, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.07778450101613998, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.046046074479818344, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.02512562833726406, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.29616284370422363, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.01005025114864111, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.8309572339057922, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.4163646101951599, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 2.449202537536621, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9648241400718689, "metrics/particle/momentum/class_7/event/energy_bias": -1.1084282398223877, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.329019069671631, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 1.0914685726165771, "metrics/particle/momentum/class_7/event/px_bias": 0.17429634928703308, "metrics/particle/momentum/class_7/event/py_bias": -0.022581566125154495, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.8336783647537231, "metrics/particle/momentum/class_7/event/pz_bias": -0.004856828600168228, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.599267959594727, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 3.8910553455352783, "step_valid_time": 4026.6139798164368, "step_total_time": 4030.505035638809, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0583746066922841, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.40007694002340244, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84715, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16445, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12938, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.20184105553850876, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19412146609219147, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19790601119200915, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8058785339078085, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15879717704817428, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1527238387534675, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15570130573440039, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.20184105553850876, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.053216076422193, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.41348079417593675, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84599, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15602, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12204, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.1920506160834082, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18442298372321186, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18815952917907838, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8155770162767881, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15022341486231983, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14425702431470822, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14717977785549755, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.1920506160834082, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0597046195357884, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.42196484009064406, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84024, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15037, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11724, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18612682419636337, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.17896077311244407, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18247346993259028, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8210392268875559, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14511876616866157, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1395315624107398, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14227033061712366, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18612682419636337} \ No newline at end of file diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_30000.json b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..5d8dc565893af63430f0707650db5e787a4752c4 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 0.1292780253212559, "train": {"Classification_binary": 3.6431329250335693, "Classification": 0.0127357617020607, "Regression_pt": 0.34053847193717957, "Regression_eta": 0.07216409593820572, "Regression_sin_phi": 0.04983179271221161, "Regression_cos_phi": 0.047406744211912155, "Regression_energy": 0.3981030583381653, "Total": 12.498800277709961, "Total_unweighted": 4.563912849873304}, "train_group": {"physical_pu": {"Classification_binary": 2.994702567111005, "Classification": 0.015743123162742498, "Regression_pt": 0.23833327650368868, "Regression_eta": 0.1065568833069487, "Regression_sin_phi": 0.04752683750056958, "Regression_cos_phi": 0.04757225255553539, "Regression_energy": 0.2837910285362831, "Total": 10.944129053052965, "Total_unweighted": 3.734225968676773}, "physical_nopu": {"Classification_binary": 3.591771700725828, "Classification": 0.014880489988295552, "Regression_pt": 0.3963299616269677, "Regression_eta": 0.0708358441412908, "Regression_sin_phi": 0.048110297988645315, "Regression_cos_phi": 0.048389894041520665, "Regression_energy": 0.4422069247012878, "Total": 13.06843902411634, "Total_unweighted": 4.612525113213836}}, "valid": {"Classification_binary": 3.2862536112467446, "Classification": 0.015306280304988226, "Regression_pt": 0.3174944917360942, "Regression_eta": 0.08925202488899231, "Regression_sin_phi": 0.04816252986590067, "Regression_cos_phi": 0.048141678174336754, "Regression_energy": 0.3636331558227539, "Total": 12.027127583821615, "group/physical_pu/Classification": 0.015709728002548218, "group/physical_pu/Classification_binary": 2.979069709777832, "group/physical_pu/Regression_cos_phi": 0.047725061575571694, "group/physical_pu/Regression_energy": 0.28235634167989093, "group/physical_pu/Regression_eta": 0.10610475142796834, "group/physical_pu/Regression_pt": 0.2362462282180786, "group/physical_pu/Regression_sin_phi": 0.04765008886655172, "group/physical_pu/Total": 10.910654703776041, "group/physical_pu/Total_unweighted": 3.7148619095484414, "group/physical_nopu/Classification": 0.014902833849191666, "group/physical_nopu/Classification_binary": 3.5934375127156577, "group/physical_nopu/Regression_cos_phi": 0.04855828980604807, "group/physical_nopu/Regression_energy": 0.44491004943847656, "group/physical_nopu/Regression_eta": 0.07239928841590881, "group/physical_nopu/Regression_pt": 0.39874263604482013, "group/physical_nopu/Regression_sin_phi": 0.04867495596408844, "group/physical_nopu/Total": 13.143601735432943, "group/physical_nopu/Total_unweighted": 4.621625566234191, "Total_unweighted": 4.168243772039811, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.015083561651408672, "diagnostic/loss/cms_pf/Classification_binary": 3.280327081680298, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.0475190170109272, "diagnostic/loss/cms_pf/Regression_energy": 0.3474291265010834, "diagnostic/loss/cms_pf/Regression_eta": 0.0894235298037529, "diagnostic/loss/cms_pf/Regression_pt": 0.30070358514785767, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.04635149985551834, "diagnostic/regression/cms_pf/energy_pred_mean": 0.2051314115524292, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.3581063449382782, "diagnostic/regression/cms_pf/energy_residual_mean": 0.0943651795387268, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5880852937698364, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.11425621807575226, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.33200374245643616, "diagnostic/regression/cms_pf/pt_residual_mean": 0.12502838671207428, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5510993599891663, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8850714564323425, "metrics/particle/class_1/pid_accuracy": 0.9776285886764526, "metrics/particle/class_2/efficiency": 0.25664547085762024, "metrics/particle/class_2/pid_accuracy": 0.20022757351398468, "metrics/particle/class_3/efficiency": 0.38857796788215637, "metrics/particle/class_3/pid_accuracy": 0.47469115257263184, "metrics/particle/class_4/efficiency": 0.43563514947891235, "metrics/particle/class_4/pid_accuracy": 0.6509593725204468, "metrics/particle/class_5/efficiency": 0.37287718057632446, "metrics/particle/class_5/pid_accuracy": 0.6207237839698792, "metrics/particle/class_6/efficiency": 0.8604222536087036, "metrics/particle/class_6/pid_accuracy": 0.054347824305295944, "metrics/particle/class_7/efficiency": 0.9610801935195923, "metrics/particle/class_7/pid_accuracy": 0.24090908467769623, "metrics/particle/count/bias_mean": 71.49739837646484, "metrics/particle/count/mae": 94.30338287353516, "metrics/particle/count/prediction_mean": 1600.1009521484375, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.09196454286575317, "metrics/particle/event/energy_response_mean": 0.9267023205757141, "metrics/particle/event/met_abs_error": 23.509780883789062, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05326806381344795, "metrics/particle/event/scalar_pt_response_mean": 1.0008668899536133, "metrics/particle/event/vector_pt_closure": 0.03148379549384117, "metrics/particle/matched/delta_eta_abs_mean": 0.020541148260235786, "metrics/particle/matched/delta_phi_abs_mean": 0.017517417669296265, "metrics/particle/matched/delta_r_mean": 0.0301546361297369, "metrics/particle/matched/energy_relative_abs_mean": 0.11487724632024765, "metrics/particle/matched/pid_accuracy": 0.843698263168335, "metrics/particle/matched/pt_relative_abs_mean": 0.1052587553858757, "metrics/particle/matching/duplicate_fraction": 0.1130446270108223, "metrics/particle/matching/efficiency": 0.6325030326843262, "metrics/particle/matching/f1": 0.6180490255355835, "metrics/particle/matching/purity": 0.6042408347129822, "metrics/particle/matching/target_coverage_dr0p05": 0.5115410089492798, "metrics/particle/matching/target_coverage_dr0p10": 0.6991139054298401, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.553626298904419, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7501034140586853, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020013127475976944, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0660107210278511, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05739203467965126, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0034138928167521954, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1077246218919754, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.001983135938644409, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.22435741126537323, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.00661286897957325, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6101269721984863, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68074.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4756621718406677, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.124483585357666, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.38049599528312683, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.25263866782188416, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14173084497451782, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.145322322845459, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.039705630391836166, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8478118181228638, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.07738500088453293, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.145881652832031, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87645.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7766482830047607, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 68.02915954589844, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 52.05720520019531, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 17.17767333984375, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.562015950679779, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.9133646488189697, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 47.10610580444336, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -5.5160932540893555, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 79.74200439453125, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.042403608560562134, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16212520003318787, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15986959636211395, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.016828587278723717, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2133062183856964, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17313608527183533, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.480984687805176, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.011488997377455235, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.594292640686035, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 120258.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5406440496444702, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9433428645133972, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9571467041969299, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.739935040473938, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3253941237926483, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3.625917673110962, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.3802070915699005, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.984406471252441, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.31545737385749817, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.91698455810547, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228486.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5198974609375, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -95.11183166503906, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 468.57159423828125, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 21.06536865234375, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 3.77463698387146, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 5.325322151184082, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 467.13275146484375, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -273.3422546386719, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 688.0020141601562, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.017003707587718964, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.07465633004903793, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.048816174268722534, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.007543465122580528, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10465821623802185, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006761667900718749, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.07921195030212402, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.011086796410381794, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7575772404670715, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59157.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3525327146053314, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.6031800508499146, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.29351571202278137, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.19456499814987183, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11010610312223434, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.889762878417969, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.023492256179451942, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.134337306022644, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.09988447278738022, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.986699104309082, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72194.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8193717002868652, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 4.901659965515137, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 25.642967224121094, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 18.273784637451172, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -4.400571823120117, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.0521023273468018, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 15.544657707214355, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.109832763671875, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 45.06334686279297, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03017360344529152, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1147804856300354, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10513728857040405, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.007313137874007225, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16708950698375702, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08483609557151794, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.743924379348755, "metrics/particle/momentum/all/accepted/pz_bias": -0.010051639750599861, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.134514808654785, "metrics/particle/momentum/all/accepted/samples": 247489.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0793942213058472, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1068975925445557, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7036201357841492, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5285617113113403, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2439168244600296, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.6358728408813477, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.23703856766223907, "metrics/particle/momentum/all/assigned/pz_abs_mean": 10.032979011535645, "metrics/particle/momentum/all/assigned/pz_bias": -0.2216469794511795, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.797086715698242, "metrics/particle/momentum/all/assigned/samples": 388325.0, "metrics/particle/momentum/all/efficiency": 0.6327052712440491, "metrics/particle/momentum/all/event/energy_bias": -22.18100357055664, "metrics/particle/momentum/all/event/p3_error_mean": 471.602783203125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 30.073474884033203, "metrics/particle/momentum/all/event/px_bias": -1.1879504919052124, "metrics/particle/momentum/all/event/py_bias": 9.290788650512695, "metrics/particle/momentum/all/event/pz_abs_mean": 467.575927734375, "metrics/particle/momentum/all/event/pz_bias": -278.7485046386719, "metrics/particle/momentum/all/event/pz_residual_rms": 697.2377319335938, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.017475467175245285, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06324965506792068, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05139457434415817, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004666955675929785, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09487425535917282, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012062938185408711, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.2099277228116989, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.01032310537993908, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1619185209274292, "metrics/particle/momentum/class_1/accepted/samples": 170771.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.23282501101493835, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.9825184345245361, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3198353350162506, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.25998789072036743, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08059630542993546, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.502214193344116, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.02303047850728035, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1594440937042236, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.010794313624501228, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.515393257141113, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.885796844959259, "metrics/particle/momentum/class_1/event/energy_bias": 47.04265594482422, "metrics/particle/momentum/class_1/event/p3_error_mean": 59.609527587890625, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 17.881458282470703, "metrics/particle/momentum/class_1/event/px_bias": -1.4658474922180176, "metrics/particle/momentum/class_1/event/py_bias": 1.7932498455047607, "metrics/particle/momentum/class_1/event/pz_abs_mean": 53.69264221191406, "metrics/particle/momentum/class_1/event/pz_bias": -19.16237449645996, "metrics/particle/momentum/class_1/event/pz_residual_rms": 88.04183197021484, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.05850575119256973, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2401004135608673, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23272159695625305, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07781800627708435, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27266228199005127, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.10984896123409271, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.174797058105469, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.11158290505409241, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 10.336074829101562, "metrics/particle/momentum/class_2/accepted/samples": 8011.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.181462526321411, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.4944682121276855, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.150397539138794, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6563027501106262, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.47568750381469727, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.517739772796631, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.3500998318195343, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.6823787689209, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.01753968559205532, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 34.94535827636719, "metrics/particle/momentum/class_2/assigned/samples": 31054.0, "metrics/particle/momentum/class_2/efficiency": 0.2578703463077545, "metrics/particle/momentum/class_2/event/energy_bias": -317.3421936035156, "metrics/particle/momentum/class_2/event/p3_error_mean": 174.97366333007812, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 29.55795669555664, "metrics/particle/momentum/class_2/event/px_bias": 4.238007068634033, "metrics/particle/momentum/class_2/event/py_bias": -0.8028179407119751, "metrics/particle/momentum/class_2/event/pz_abs_mean": 168.6397247314453, "metrics/particle/momentum/class_2/event/pz_bias": 7.874958515167236, "metrics/particle/momentum/class_2/event/pz_residual_rms": 235.2459259033203, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.06079454720020294, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.22765550017356873, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.2242954522371292, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.005853550508618355, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.26399511098861694, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.31512340903282166, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.360391616821289, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.00777394138276577, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 13.964994430541992, "metrics/particle/momentum/class_3/accepted/samples": 20703.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.0776541233062744, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.9476155042648315, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.0936532020568848, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.9168073534965515, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4292602837085724, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 2.6224420070648193, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5426267981529236, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.655548095703125, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.4550836384296417, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.86189651489258, "metrics/particle/momentum/class_3/assigned/samples": 52467.0, "metrics/particle/momentum/class_3/efficiency": 0.38841673731803894, "metrics/particle/momentum/class_3/event/energy_bias": 747.2578735351562, "metrics/particle/momentum/class_3/event/p3_error_mean": 306.8424377441406, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.054452896118164, "metrics/particle/momentum/class_3/event/px_bias": 0.4608916640281677, "metrics/particle/momentum/class_3/event/py_bias": 1.9235543012619019, "metrics/particle/momentum/class_3/event/pz_abs_mean": 306.4653625488281, "metrics/particle/momentum/class_3/event/pz_bias": -77.252685546875, "metrics/particle/momentum/class_3/event/pz_residual_rms": 474.3704833984375, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.059695079922676086, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.23281767964363098, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22971688210964203, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.05894448980689049, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.26842325925827026, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.359096497297287, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 11.004300117492676, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.016074519604444504, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.375621795654297, "metrics/particle/momentum/class_4/accepted/samples": 36525.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.8886497020721436, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0478304624557495, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0883557796478271, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8119169473648071, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3740917146205902, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.5583109855651855, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5338647365570068, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.878822326660156, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.698486328125, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 41.27840805053711, "metrics/particle/momentum/class_4/assigned/samples": 82298.0, "metrics/particle/momentum/class_4/efficiency": 0.43335628509521484, "metrics/particle/momentum/class_4/event/energy_bias": -473.2479553222656, "metrics/particle/momentum/class_4/event/p3_error_mean": 369.38275146484375, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 11.352733612060547, "metrics/particle/momentum/class_4/event/px_bias": 1.929606318473816, "metrics/particle/momentum/class_4/event/py_bias": 1.0542761087417603, "metrics/particle/momentum/class_4/event/pz_abs_mean": 368.58526611328125, "metrics/particle/momentum/class_4/event/pz_bias": -193.0615234375, "metrics/particle/momentum/class_4/event/pz_residual_rms": 566.7015991210938, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.05219104886054993, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22338519990444183, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2123858630657196, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.028865741565823555, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2533598840236664, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.024116501212120056, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.262120246887207, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.08639191836118698, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.525051116943359, "metrics/particle/momentum/class_5/accepted/samples": 10781.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.447772741317749, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 1.9919782876968384, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9894224405288696, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6794285178184509, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.38139164447784424, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 5.02388858795166, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.14870093762874603, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.447686195373535, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.20908619463443756, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 17.814403533935547, "metrics/particle/momentum/class_5/assigned/samples": 28944.0, "metrics/particle/momentum/class_5/efficiency": 0.37243929505348206, "metrics/particle/momentum/class_5/event/energy_bias": -19.51219940185547, "metrics/particle/momentum/class_5/event/p3_error_mean": 115.75536346435547, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 27.35051918029785, "metrics/particle/momentum/class_5/event/px_bias": -4.069952011108398, "metrics/particle/momentum/class_5/event/py_bias": 3.4760375022888184, "metrics/particle/momentum/class_5/event/pz_abs_mean": 106.98249816894531, "metrics/particle/momentum/class_5/event/pz_bias": 2.6916465759277344, "metrics/particle/momentum/class_5/event/pz_residual_rms": 170.74559020996094, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.02052225172519684, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.09631478786468506, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.09590889513492584, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.04517977312207222, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.1375555545091629, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.013861386105418205, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.7096290588378906, "metrics/particle/momentum/class_6/accepted/pz_bias": 0.19889505207538605, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 2.6782028675079346, "metrics/particle/momentum/class_6/accepted/samples": 505.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.30570536851882935, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 0.6734799742698669, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.43318915367126465, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.2763425409793854, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.09739130735397339, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 1.695647954940796, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.03999999910593033, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 1.9439493417739868, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.2724605202674866, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 8.382925033569336, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8782608509063721, "metrics/particle/momentum/class_6/event/energy_bias": -4.994778633117676, "metrics/particle/momentum/class_6/event/p3_error_mean": 11.954488754272461, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 6.268063068389893, "metrics/particle/momentum/class_6/event/px_bias": -2.2908859252929688, "metrics/particle/momentum/class_6/event/py_bias": 1.8523831367492676, "metrics/particle/momentum/class_6/event/pz_abs_mean": 9.08222770690918, "metrics/particle/momentum/class_6/event/pz_bias": 0.44302159547805786, "metrics/particle/momentum/class_6/event/pz_residual_rms": 35.0975227355957, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.013545137830078602, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.04396509379148483, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.036899734288454056, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.011637620627880096, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.06384626030921936, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.015544041059911251, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.5787492990493774, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.20548322796821594, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 2.021928071975708, "metrics/particle/momentum/class_7/accepted/samples": 193.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.03135867789387703, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.06556636095046997, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.05664213374257088, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.024086937308311462, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.02512562833726406, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.14445868134498596, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.015075377188622952, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.6009427905082703, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.19121339917182922, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 2.0226902961730957, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9698492288589478, "metrics/particle/momentum/class_7/event/energy_bias": -1.3843824863433838, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.251737117767334, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 1.0264681577682495, "metrics/particle/momentum/class_7/event/px_bias": 0.010229978710412979, "metrics/particle/momentum/class_7/event/py_bias": -0.005893982946872711, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.8118956089019775, "metrics/particle/momentum/class_7/event/pz_bias": -0.28155162930488586, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.759368896484375, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 3.1479060649871826, "step_valid_time": 6153.184191226959, "step_total_time": 6156.332097530365, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0471341521235313, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.378433912142911, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84125, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16138, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12939, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19807302853636086, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.1918335809806835, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19490338164251209, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8081664190193165, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15880945075176434, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1538068350668648, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.156268115942029, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19807302853636086, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.04405815175934, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3955166176738064, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 83964, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15412, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12155, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18971183791036325, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18355485684340908, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18658256811316987, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8164451431565909, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14962025628085032, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14476442284788718, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14715229142327924, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18971183791036325, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0465181702481519, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.4072959807036549, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 83569, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 14945, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11734, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18498805530455878, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.17883425672199021, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18185911242531547, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8211657432780098, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14524254539603162, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14041091792411062, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14278586986943137, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18498805530455878} \ No newline at end of file diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_40000.json b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..c990ee02c930226161131dfe30b220ac77f4f5c3 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 0.17237070042834118, "train": {"Classification_binary": 3.5708093643188477, "Classification": 0.012191655114293098, "Regression_pt": 0.5553004741668701, "Regression_eta": 0.07466916739940643, "Regression_sin_phi": 0.04639614373445511, "Regression_cos_phi": 0.04589694365859032, "Regression_energy": 0.5954195261001587, "Total": 14.756237983703613, "Total_unweighted": 4.900683274492621}, "train_group": {"physical_pu": {"Classification_binary": 2.9700621327201087, "Classification": 0.01573734606646902, "Regression_pt": 0.23424383408420688, "Regression_eta": 0.10586133484656994, "Regression_sin_phi": 0.047385072642630276, "Regression_cos_phi": 0.047363227704069116, "Regression_energy": 0.2799897835804866, "Total": 10.851692576984783, "Total_unweighted": 3.7006427316445407}, "physical_nopu": {"Classification_binary": 3.5493309390295718, "Classification": 0.014712276177970555, "Regression_pt": 0.39059537326214205, "Regression_eta": 0.07035032552805277, "Regression_sin_phi": 0.048034675095570495, "Regression_cos_phi": 0.048159302448970646, "Regression_energy": 0.4367032872371548, "Total": 12.938419774289441, "Total_unweighted": 4.557886178779433}}, "valid": {"Classification_binary": 3.263694445292155, "Classification": 0.015217152734597525, "Regression_pt": 0.3138403296470642, "Regression_eta": 0.08890992403030396, "Regression_sin_phi": 0.04804997146129608, "Regression_cos_phi": 0.04795230428377787, "Regression_energy": 0.35901427268981934, "Total": 11.939679463704428, "group/physical_pu/Classification": 0.015650687118371327, "group/physical_pu/Classification_binary": 2.968634287516276, "group/physical_pu/Regression_cos_phi": 0.04758670926094055, "group/physical_pu/Regression_energy": 0.2803141077359517, "group/physical_pu/Regression_eta": 0.10589230060577393, "group/physical_pu/Regression_pt": 0.23484853903452554, "group/physical_pu/Regression_sin_phi": 0.047573273380597435, "group/physical_pu/Total": 10.869064331054688, "group/physical_pu/Total_unweighted": 3.7004999046524363, "group/physical_nopu/Classification": 0.014783620834350586, "group/physical_nopu/Classification_binary": 3.5587539672851562, "group/physical_nopu/Regression_cos_phi": 0.04831789433956146, "group/physical_nopu/Regression_energy": 0.4377143780390422, "group/physical_nopu/Regression_eta": 0.07192755738894145, "group/physical_nopu/Regression_pt": 0.3928321599960327, "group/physical_nopu/Regression_sin_phi": 0.048526664574941, "group/physical_nopu/Total": 13.010294596354166, "group/physical_nopu/Total_unweighted": 4.572856242458026, "Total_unweighted": 4.136678400139013, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.014996262267231941, "diagnostic/loss/cms_pf/Classification_binary": 3.2567315101623535, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.047302160412073135, "diagnostic/loss/cms_pf/Regression_energy": 0.344427227973938, "diagnostic/loss/cms_pf/Regression_eta": 0.08892977237701416, "diagnostic/loss/cms_pf/Regression_pt": 0.2994489073753357, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.04620146006345749, "diagnostic/regression/cms_pf/energy_pred_mean": 0.22005118429660797, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.36083367466926575, "diagnostic/regression/cms_pf/energy_residual_mean": 0.10928494483232498, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5896862149238586, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.10780257731676102, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.3303394615650177, "diagnostic/regression/cms_pf/pt_residual_mean": 0.11857476085424423, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5476004481315613, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8843146562576294, "metrics/particle/class_1/pid_accuracy": 0.9772791266441345, "metrics/particle/class_2/efficiency": 0.2542128562927246, "metrics/particle/class_2/pid_accuracy": 0.20283913612365723, "metrics/particle/class_3/efficiency": 0.388963520526886, "metrics/particle/class_3/pid_accuracy": 0.46309351921081543, "metrics/particle/class_4/efficiency": 0.4267999529838562, "metrics/particle/class_4/pid_accuracy": 0.6512650847434998, "metrics/particle/class_5/efficiency": 0.36330142617225647, "metrics/particle/class_5/pid_accuracy": 0.6132199764251709, "metrics/particle/class_6/efficiency": 0.8524869084358215, "metrics/particle/class_6/pid_accuracy": 0.04920212924480438, "metrics/particle/class_7/efficiency": 0.9602859616279602, "metrics/particle/class_7/pid_accuracy": 0.21712158620357513, "metrics/particle/count/bias_mean": 61.19921875, "metrics/particle/count/mae": 86.7734375, "metrics/particle/count/prediction_mean": 1589.802734375, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.09517853707075119, "metrics/particle/event/energy_response_mean": 0.9279007911682129, "metrics/particle/event/met_abs_error": 21.19899559020996, "metrics/particle/event/scalar_pt_relative_abs_error": 0.048296887427568436, "metrics/particle/event/scalar_pt_response_mean": 0.990368127822876, "metrics/particle/event/vector_pt_closure": 0.029878608882427216, "metrics/particle/matched/delta_eta_abs_mean": 0.022716155275702477, "metrics/particle/matched/delta_phi_abs_mean": 0.017545565962791443, "metrics/particle/matched/delta_r_mean": 0.0320916548371315, "metrics/particle/matched/energy_relative_abs_mean": 0.11914577335119247, "metrics/particle/matched/pid_accuracy": 0.8431945443153381, "metrics/particle/matched/pt_relative_abs_mean": 0.1058678850531578, "metrics/particle/matching/duplicate_fraction": 0.11416592448949814, "metrics/particle/matching/efficiency": 0.6293672919273376, "metrics/particle/matching/f1": 0.6170158386230469, "metrics/particle/matching/purity": 0.6051399111747742, "metrics/particle/matching/target_coverage_dr0p05": 0.500597357749939, "metrics/particle/matching/target_coverage_dr0p10": 0.6941039562225342, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5530346632003784, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7505061030387878, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.023716596886515617, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0727454423904419, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05909697711467743, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.002743571065366268, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.10946521908044815, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0019579266663640738, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.2223139852285385, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.009584209881722927, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.361868977546692, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 67929.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.490771621465683, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1690078973770142, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.37840110063552856, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.24446488916873932, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14048540592193604, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.3446390628814697, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.043199945241212845, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.9352015256881714, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.12789078056812286, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.554956436157227, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87639.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7749940156936646, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 33.57335662841797, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 46.36008071899414, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 16.75887107849121, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 3.422360897064209, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.6125059127807617, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 41.4575080871582, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 7.8524861335754395, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 62.22708511352539, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04431905224919319, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16390810906887054, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16013053059577942, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.018817458301782608, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21347425878047943, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.1758357286453247, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.527542591094971, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 6.479011790361255e-05, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.663046836853027, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 118969.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5467040538787842, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0216314792633057, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9618843197822571, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7395621538162231, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.32443827390670776, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 4.507246017456055, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.3839319348335266, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 16.13783836364746, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.22947657108306885, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 33.07467269897461, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228142.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5143248438835144, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 141.5834503173828, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 441.9581298828125, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 20.490407943725586, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -2.684905529022217, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -3.0780625343322754, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 440.44195556640625, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -257.1128234863281, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 659.1532592773438, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.016868088394403458, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08180584013462067, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.05064249411225319, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010822317562997341, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10569269955158234, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007752460660412908, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.08186844736337662, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.006546352058649063, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8053663969039917, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59336.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3526113033294678, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.4238337278366089, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.2690945863723755, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.1683124154806137, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10516546666622162, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 3.764622926712036, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.024324361234903336, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.1454318761825562, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.09009448438882828, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 7.046112537384033, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72191.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8218510150909424, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.4065871834754944, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 22.09476089477539, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 15.560954093933105, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 5.002377033233643, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.8493272662162781, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 13.182530403137207, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.4110386371612549, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 30.05718231201172, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0320204421877861, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11897440254688263, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1058744341135025, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.007240711711347103, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16737839579582214, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08568272739648819, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.751713514328003, "metrics/particle/momentum/all/accepted/pz_bias": -0.004190210253000259, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.147994995117188, "metrics/particle/momentum/all/accepted/samples": 246234.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0859915018081665, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1297610998153687, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7011720538139343, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5214306116104126, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.24208447337150574, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.978067398071289, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.24005082249641418, "metrics/particle/momentum/all/assigned/pz_abs_mean": 10.13992691040039, "metrics/particle/momentum/all/assigned/pz_bias": -0.18059413135051727, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.944849014282227, "metrics/particle/momentum/all/assigned/samples": 387972.0, "metrics/particle/momentum/all/efficiency": 0.6294968724250793, "metrics/particle/momentum/all/event/energy_bias": 174.750244140625, "metrics/particle/momentum/all/event/p3_error_mean": 436.7352600097656, "metrics/particle/momentum/all/event/pt_vector_error_mean": 26.833728790283203, "metrics/particle/momentum/all/event/px_bias": 5.739831924438477, "metrics/particle/momentum/all/event/py_bias": -2.3148837089538574, "metrics/particle/momentum/all/event/pz_abs_mean": 433.88970947265625, "metrics/particle/momentum/all/event/pz_bias": -247.84930419921875, "metrics/particle/momentum/all/event/pz_residual_rms": 654.01416015625, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02013247273862362, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.0698685497045517, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05294199660420418, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0057061221450567245, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0962386354804039, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013427698286250234, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.2142755687236786, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.013147319667041302, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1082179546356201, "metrics/particle/momentum/class_1/accepted/samples": 170543.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.23936520516872406, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.9555951952934265, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.33225828409194946, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.270599901676178, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07953814417123795, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.728358507156372, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.02428055740892887, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.2117236852645874, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.04098362475633621, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.8972554206848145, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8846141695976257, "metrics/particle/momentum/class_1/event/energy_bias": 67.30034637451172, "metrics/particle/momentum/class_1/event/p3_error_mean": 56.97767639160156, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 17.95798110961914, "metrics/particle/momentum/class_1/event/px_bias": 3.226874828338623, "metrics/particle/momentum/class_1/event/py_bias": 0.040881842374801636, "metrics/particle/momentum/class_1/event/pz_abs_mean": 50.943321228027344, "metrics/particle/momentum/class_1/event/pz_bias": -12.859596252441406, "metrics/particle/momentum/class_1/event/pz_residual_rms": 82.46072387695312, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.05906148999929428, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23960886895656586, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2319076657295227, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07669133692979813, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2727307081222534, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.10906568914651871, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.161664009094238, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.019858427345752716, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 9.89720344543457, "metrics/particle/momentum/class_2/accepted/samples": 7931.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.1921913623809814, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.5221714973449707, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1141682863235474, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.615217387676239, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.47319069504737854, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.643784046173096, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.35341882705688477, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.647811889648438, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.23973417282104492, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 34.7972412109375, "metrics/particle/momentum/class_2/assigned/samples": 31034.0, "metrics/particle/momentum/class_2/efficiency": 0.25529518723487854, "metrics/particle/momentum/class_2/event/energy_bias": -272.8325500488281, "metrics/particle/momentum/class_2/event/p3_error_mean": 180.1333465576172, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 28.626924514770508, "metrics/particle/momentum/class_2/event/px_bias": -0.11727359890937805, "metrics/particle/momentum/class_2/event/py_bias": -3.1121408939361572, "metrics/particle/momentum/class_2/event/pz_abs_mean": 173.73858642578125, "metrics/particle/momentum/class_2/event/pz_bias": -68.33466339111328, "metrics/particle/momentum/class_2/event/pz_residual_rms": 249.23231506347656, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.06079471856355667, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.22744260728359222, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.22490760684013367, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.010007908567786217, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2641984522342682, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.32192274928092957, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.518771171569824, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.04080311954021454, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 14.129573822021484, "metrics/particle/momentum/class_3/accepted/samples": 20741.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.0567617416381836, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.004374623298645, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.08851158618927, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.9071861505508423, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4243568480014801, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 3.213404655456543, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5455308556556702, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.76230812072754, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.38312193751335144, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 34.05039978027344, "metrics/particle/momentum/class_3/assigned/samples": 52437.0, "metrics/particle/momentum/class_3/efficiency": 0.3891296684741974, "metrics/particle/momentum/class_3/event/energy_bias": 906.028564453125, "metrics/particle/momentum/class_3/event/p3_error_mean": 322.0340576171875, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.802669525146484, "metrics/particle/momentum/class_3/event/px_bias": 0.9910997748374939, "metrics/particle/momentum/class_3/event/py_bias": -0.9266294836997986, "metrics/particle/momentum/class_3/event/pz_abs_mean": 321.6021423339844, "metrics/particle/momentum/class_3/event/pz_bias": -160.92588806152344, "metrics/particle/momentum/class_3/event/pz_residual_rms": 517.2124633789062, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.0602729432284832, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.23218277096748352, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22993361949920654, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.06100818142294884, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2689572274684906, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.3652142286300659, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 11.091257095336914, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.06939993798732758, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.507692337036133, "metrics/particle/momentum/class_4/accepted/samples": 35779.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.904036521911621, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.098360538482666, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.050659418106079, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.7651253342628479, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3725427985191345, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.3268558979034424, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5401443839073181, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 23.147615432739258, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.3537590801715851, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 41.48666763305664, "metrics/particle/momentum/class_4/assigned/samples": 82004.0, "metrics/particle/momentum/class_4/efficiency": 0.42450523376464844, "metrics/particle/momentum/class_4/event/energy_bias": -490.4198303222656, "metrics/particle/momentum/class_4/event/p3_error_mean": 337.67877197265625, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 11.611647605895996, "metrics/particle/momentum/class_4/event/px_bias": -2.151264190673828, "metrics/particle/momentum/class_4/event/py_bias": -1.9493401050567627, "metrics/particle/momentum/class_4/event/pz_abs_mean": 337.08917236328125, "metrics/particle/momentum/class_4/event/pz_bias": 3.3787965774536133, "metrics/particle/momentum/class_4/event/pz_residual_rms": 497.02569580078125, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.05209114030003548, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22699876129627228, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21365201473236084, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02474396862089634, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.25432610511779785, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02370567061007023, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2567830085754395, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.023034734651446342, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.857359886169434, "metrics/particle/momentum/class_5/accepted/samples": 10546.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4851415157318115, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.193622589111328, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 1.0321052074432373, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.709135115146637, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.381544828414917, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 6.803393363952637, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.15735268592834473, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.753246307373047, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.18754658102989197, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 18.187053680419922, "metrics/particle/momentum/class_5/assigned/samples": 28935.0, "metrics/particle/momentum/class_5/efficiency": 0.3643209934234619, "metrics/particle/momentum/class_5/event/energy_bias": -25.41569709777832, "metrics/particle/momentum/class_5/event/p3_error_mean": 114.48596954345703, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 26.153064727783203, "metrics/particle/momentum/class_5/event/px_bias": 4.265480041503906, "metrics/particle/momentum/class_5/event/py_bias": 4.302125453948975, "metrics/particle/momentum/class_5/event/pz_abs_mean": 106.12934112548828, "metrics/particle/momentum/class_5/event/pz_bias": -9.433398246765137, "metrics/particle/momentum/class_5/event/pz_residual_rms": 167.49888610839844, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.024022024124860764, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.098977230489254, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.09776756912469864, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.04394194483757019, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.13970695436000824, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.01796407252550125, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.7633439302444458, "metrics/particle/momentum/class_6/accepted/pz_bias": 0.016738753765821457, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 2.7039694786071777, "metrics/particle/momentum/class_6/accepted/samples": 501.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.3263380825519562, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 1.0862051248550415, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.5007096529006958, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.34538349509239197, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.09739130735397339, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 1.9397209882736206, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.052173912525177, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 2.2866501808166504, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.28982797265052795, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 9.067389488220215, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8713043332099915, "metrics/particle/momentum/class_6/event/energy_bias": -8.12272834777832, "metrics/particle/momentum/class_6/event/p3_error_mean": 8.953807830810547, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 3.254956007003784, "metrics/particle/momentum/class_6/event/px_bias": -0.5093610882759094, "metrics/particle/momentum/class_6/event/py_bias": -0.659982442855835, "metrics/particle/momentum/class_6/event/pz_abs_mean": 7.841311931610107, "metrics/particle/momentum/class_6/event/pz_bias": 0.3459952771663666, "metrics/particle/momentum/class_6/event/pz_residual_rms": 24.93433952331543, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.019773514941334724, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.05928068608045578, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.04135111719369888, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.018866587430238724, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.07450256496667862, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.005181347019970417, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.6046072244644165, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.17780454456806183, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 2.0690524578094482, "metrics/particle/momentum/class_7/accepted/samples": 193.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.02935260534286499, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.08035356551408768, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.07126251608133316, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.04293948784470558, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.02512562833726406, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.2325911670923233, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.005025125574320555, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.6214389801025391, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.15913935005664825, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 2.0663211345672607, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9698492288589478, "metrics/particle/momentum/class_7/event/energy_bias": -1.787867784500122, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.2556240558624268, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 1.0122880935668945, "metrics/particle/momentum/class_7/event/px_bias": 0.03427627682685852, "metrics/particle/momentum/class_7/event/py_bias": -0.009797666221857071, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.8307138681411743, "metrics/particle/momentum/class_7/event/pz_bias": -0.02055644989013672, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.807620525360107, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 3.001978635787964, "step_valid_time": 8163.877256155014, "step_total_time": 8166.879235267639, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.034427199827121, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3636866960848618, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84709, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16860, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13589, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.20693464252838295, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19903434109716794, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.20290762046887786, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.800965658902832, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.16678735808530223, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.16041979010494753, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.1635416165214461, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.20693464252838295, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0348796999023928, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.38220299040403694, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84450, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15967, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12690, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19654353204741565, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.1890704558910598, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.1927345810524537, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8109295441089402, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15620576324179272, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1502664298401421, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15317854534700553, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19654353204741565, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0400349280516166, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.4036355194294401, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 83848, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15210, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12008, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18826820482986545, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18139967560347295, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.1847701306510687, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8186003243965271, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14863409622597135, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14321152561778455, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.1458724345074315, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18826820482986545} \ No newline at end of file diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_50000.json b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..0788c4a03e0fb481e011250be8a553d1748c987d --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 0.2154633755354265, "train": {"Classification_binary": 3.4424147605895996, "Classification": 0.01589777320623398, "Regression_pt": 0.45725512504577637, "Regression_eta": 0.0683407261967659, "Regression_sin_phi": 0.04636869579553604, "Regression_cos_phi": 0.04259190335869789, "Regression_energy": 0.5013267993927002, "Total": 13.365400314331055, "Total_unweighted": 4.57419578358531}, "train_group": {"physical_pu": {"Classification_binary": 2.9565212412194892, "Classification": 0.015587834109153067, "Regression_pt": 0.23207483481574845, "Regression_eta": 0.10542395972943568, "Regression_sin_phi": 0.047347324030412426, "Regression_cos_phi": 0.04730799691853942, "Regression_energy": 0.2770941097002763, "Total": 10.797196388244629, "Total_unweighted": 3.6813573005230547}, "physical_nopu": {"Classification_binary": 3.521162594636806, "Classification": 0.014626817653781444, "Regression_pt": 0.38323910145869744, "Regression_eta": 0.06947089998897808, "Regression_sin_phi": 0.04784381444534191, "Regression_cos_phi": 0.047947851147982154, "Regression_energy": 0.42889715854090826, "Total": 12.794195285331298, "Total_unweighted": 4.5131882378724955}}, "valid": {"Classification_binary": 3.236013412475586, "Classification": 0.015170596539974213, "Regression_pt": 0.30924129486083984, "Regression_eta": 0.08849118153254192, "Regression_sin_phi": 0.04785735408465067, "Regression_cos_phi": 0.047804961601893105, "Regression_energy": 0.3543286720911662, "Total": 11.838942209879557, "group/physical_pu/Classification": 0.01563676322499911, "group/physical_pu/Classification_binary": 2.9453560511271157, "group/physical_pu/Regression_cos_phi": 0.04749436676502228, "group/physical_pu/Regression_energy": 0.27743107080459595, "group/physical_pu/Regression_eta": 0.10540548960367839, "group/physical_pu/Regression_pt": 0.23253009716669717, "group/physical_pu/Regression_sin_phi": 0.04745963712533315, "group/physical_pu/Total": 10.8023681640625, "group/physical_pu/Total_unweighted": 3.671313475817442, "group/physical_nopu/Classification": 0.014704428613185883, "group/physical_nopu/Classification_binary": 3.5266701380411782, "group/physical_nopu/Regression_cos_phi": 0.0481155663728714, "group/physical_nopu/Regression_energy": 0.4312262535095215, "group/physical_nopu/Regression_eta": 0.07157687346140544, "group/physical_nopu/Regression_pt": 0.3859524726867676, "group/physical_nopu/Regression_sin_phi": 0.0482550710439682, "group/physical_nopu/Total": 12.875516255696615, "group/physical_nopu/Total_unweighted": 4.526500803728898, "Total_unweighted": 4.098907473186652, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.014933686703443527, "diagnostic/loss/cms_pf/Classification_binary": 3.227829933166504, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.04715002328157425, "diagnostic/loss/cms_pf/Regression_energy": 0.3416123390197754, "diagnostic/loss/cms_pf/Regression_eta": 0.0886719673871994, "diagnostic/loss/cms_pf/Regression_pt": 0.2968401312828064, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.0460086315870285, "diagnostic/regression/cms_pf/energy_pred_mean": 0.21951226890087128, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.35821425914764404, "diagnostic/regression/cms_pf/energy_residual_mean": 0.10874605178833008, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5888749957084656, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.11786287277936935, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.3310337960720062, "diagnostic/regression/cms_pf/pt_residual_mean": 0.12863504886627197, "diagnostic/regression/cms_pf/pt_residual_rms": 0.549802303314209, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8857062458992004, "metrics/particle/class_1/pid_accuracy": 0.9783486127853394, "metrics/particle/class_2/efficiency": 0.25887447595596313, "metrics/particle/class_2/pid_accuracy": 0.20583060383796692, "metrics/particle/class_3/efficiency": 0.3949612081050873, "metrics/particle/class_3/pid_accuracy": 0.4785560667514801, "metrics/particle/class_4/efficiency": 0.44084426760673523, "metrics/particle/class_4/pid_accuracy": 0.6567733287811279, "metrics/particle/class_5/efficiency": 0.3739030659198761, "metrics/particle/class_5/pid_accuracy": 0.6268911957740784, "metrics/particle/class_6/efficiency": 0.8602805733680725, "metrics/particle/class_6/pid_accuracy": 0.05616867169737816, "metrics/particle/class_7/efficiency": 0.9618744850158691, "metrics/particle/class_7/pid_accuracy": 0.24855491518974304, "metrics/particle/count/bias_mean": 73.60872650146484, "metrics/particle/count/mae": 94.22396087646484, "metrics/particle/count/prediction_mean": 1602.2122802734375, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.07403045147657394, "metrics/particle/event/energy_response_mean": 0.9723596572875977, "metrics/particle/event/met_abs_error": 21.80121612548828, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05117127671837807, "metrics/particle/event/scalar_pt_response_mean": 1.0224745273590088, "metrics/particle/event/vector_pt_closure": 0.02969387173652649, "metrics/particle/matched/delta_eta_abs_mean": 0.01935659907758236, "metrics/particle/matched/delta_phi_abs_mean": 0.016936900094151497, "metrics/particle/matched/delta_r_mean": 0.028738342225551605, "metrics/particle/matched/energy_relative_abs_mean": 0.11546225845813751, "metrics/particle/matched/pid_accuracy": 0.8448609709739685, "metrics/particle/matched/pt_relative_abs_mean": 0.10538887977600098, "metrics/particle/matching/duplicate_fraction": 0.11138509958982468, "metrics/particle/matching/efficiency": 0.6350706219673157, "metrics/particle/matching/f1": 0.6201394200325012, "metrics/particle/matching/purity": 0.6058942079544067, "metrics/particle/matching/target_coverage_dr0p05": 0.5131372809410095, "metrics/particle/matching/target_coverage_dr0p10": 0.7016597390174866, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5554587244987488, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7526339292526245, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01812274381518364, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.067232146859169, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.057955704629421234, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.005393961910158396, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.10947541892528534, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0019909527618438005, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.22404536604881287, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.004841347225010395, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4319201707839966, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68309.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4720143973827362, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.18433678150177, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.39640286564826965, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.26923924684524536, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1400374174118042, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.707655906677246, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03995527699589729, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8401923179626465, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.05907939746975899, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.125904083251953, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87648.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7793293595314026, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 74.9918212890625, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 46.6518669128418, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 15.997821807861328, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.6890475749969482, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -1.2286041975021362, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 41.847171783447266, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.8609379529953003, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 67.2259521484375, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04112190008163452, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16112147271633148, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15872478485107422, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.01603260077536106, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2120637446641922, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.1741141676902771, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.520236492156982, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.03085433319211006, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.927123069763184, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 120846.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5255390405654907, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9791606068611145, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9747973084449768, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7613589763641357, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.32292646169662476, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 4.140881538391113, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.3785254955291748, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.788492202758789, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.07336997240781784, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.2150993347168, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228727.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5224394798278809, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 571.8801879882812, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 390.49871826171875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 20.396663665771484, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 3.852644920349121, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.5528112649917603, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 388.2487487792969, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 97.78997039794922, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 582.4744873046875, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.015179388225078583, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.07744766026735306, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.050465792417526245, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0063080135732889175, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10754397511482239, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006420871359296143, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.08157555013895035, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.004427401814609766, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8218533396720886, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59182.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3445042669773102, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3357235193252563, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.2738816440105438, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.1771101951599121, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1104462742805481, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 2.2283811569213867, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.023767970502376556, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0985488891601562, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.08696708083152771, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.394330978393555, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72198.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8197180032730103, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 14.608713150024414, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 22.805830001831055, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 17.416378021240234, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 3.4295496940612793, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -3.0421714782714844, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 12.75350284576416, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.07789386808872223, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 27.733776092529297, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.028613174334168434, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11535514891147614, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10520711541175842, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.007782211061567068, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.167142853140831, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08542826771736145, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.767331123352051, "metrics/particle/momentum/all/accepted/pz_bias": 0.017401164397597313, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.363602638244629, "metrics/particle/momentum/all/accepted/samples": 248337.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0684623718261719, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0916913747787476, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7141003012657166, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5417994856834412, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.24219387769699097, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.559433698654175, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.23624132573604584, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.912826538085938, "metrics/particle/momentum/all/assigned/pz_bias": 0.013703099451959133, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.24427604675293, "metrics/particle/momentum/all/assigned/samples": 388573.0, "metrics/particle/momentum/all/efficiency": 0.6348732113838196, "metrics/particle/momentum/all/event/energy_bias": 661.4807739257812, "metrics/particle/momentum/all/event/p3_error_mean": 395.3887939453125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 29.22286605834961, "metrics/particle/momentum/all/event/px_bias": 6.593148231506348, "metrics/particle/momentum/all/event/py_bias": -3.7179644107818604, "metrics/particle/momentum/all/event/pz_abs_mean": 390.5389099121094, "metrics/particle/momentum/all/event/pz_bias": 99.72879028320312, "metrics/particle/momentum/all/event/pz_residual_rms": 582.1754760742188, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.015128036960959435, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.0627858117222786, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05147145688533783, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0017372462898492813, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09541851282119751, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012181266210973263, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.2063182294368744, "metrics/particle/momentum/class_1/accepted/pz_bias": 9.277403296437114e-05, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1651188135147095, "metrics/particle/momentum/class_1/accepted/samples": 170754.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.23184679448604584, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.9142671227455139, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3374655246734619, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.27536386251449585, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08032657951116562, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.282679557800293, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.023497313261032104, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1651870012283325, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.03249429911375046, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.489236831665039, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8857086300849915, "metrics/particle/momentum/class_1/event/energy_bias": 49.337425231933594, "metrics/particle/momentum/class_1/event/p3_error_mean": 56.22993469238281, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 17.4781551361084, "metrics/particle/momentum/class_1/event/px_bias": -1.9977415800094604, "metrics/particle/momentum/class_1/event/py_bias": -3.649746894836426, "metrics/particle/momentum/class_1/event/pz_abs_mean": 50.293540954589844, "metrics/particle/momentum/class_1/event/pz_bias": -6.419393062591553, "metrics/particle/momentum/class_1/event/pz_residual_rms": 80.40103912353516, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.05809571221470833, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2372555285692215, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23164363205432892, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07472758740186691, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27174893021583557, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.1063697412610054, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.168974876403809, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.18343107402324677, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 9.940017700195312, "metrics/particle/momentum/class_2/accepted/samples": 8038.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.200882911682129, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.5307247638702393, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1483728885650635, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6623382568359375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.47155413031578064, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.241417407989502, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.35284459590911865, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.74336051940918, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.14623212814331055, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 34.873130798339844, "metrics/particle/momentum/class_2/assigned/samples": 31059.0, "metrics/particle/momentum/class_2/efficiency": 0.2587394714355469, "metrics/particle/momentum/class_2/event/energy_bias": -324.6517333984375, "metrics/particle/momentum/class_2/event/p3_error_mean": 175.99844360351562, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 28.44395637512207, "metrics/particle/momentum/class_2/event/px_bias": 3.1792469024658203, "metrics/particle/momentum/class_2/event/py_bias": -1.6663553714752197, "metrics/particle/momentum/class_2/event/pz_abs_mean": 170.111572265625, "metrics/particle/momentum/class_2/event/pz_bias": 49.012203216552734, "metrics/particle/momentum/class_2/event/pz_residual_rms": 240.247802734375, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.060883186757564545, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.2261834591627121, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.22291171550750732, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.003213145537301898, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.26242944598197937, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.31547507643699646, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.36158561706543, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0308486670255661, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 13.966912269592285, "metrics/particle/momentum/class_3/accepted/samples": 21092.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.031186103820801, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.970079779624939, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.111826777458191, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.9378911852836609, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4232754707336426, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 3.2014553546905518, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5373097658157349, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.38871192932129, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0030693081207573414, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.4394645690918, "metrics/particle/momentum/class_3/assigned/samples": 52493.0, "metrics/particle/momentum/class_3/efficiency": 0.3957149088382721, "metrics/particle/momentum/class_3/event/energy_bias": 712.6223754882812, "metrics/particle/momentum/class_3/event/p3_error_mean": 288.1435241699219, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.82120132446289, "metrics/particle/momentum/class_3/event/px_bias": 0.7804490923881531, "metrics/particle/momentum/class_3/event/py_bias": -1.2176176309585571, "metrics/particle/momentum/class_3/event/pz_abs_mean": 287.5389404296875, "metrics/particle/momentum/class_3/event/pz_bias": 14.137887001037598, "metrics/particle/momentum/class_3/event/pz_residual_rms": 457.0618896484375, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.05971093475818634, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.22947415709495544, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22677133977413177, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.05430774390697479, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2659240663051605, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.35942336916923523, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 11.035386085510254, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.11144203692674637, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 18.016183853149414, "metrics/particle/momentum/class_4/accepted/samples": 36834.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.8680617809295654, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.083162784576416, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0841461420059204, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8160152435302734, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3702402114868164, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.3491013050079346, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5303463935852051, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.413597106933594, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.015227535739541054, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 39.897281646728516, "metrics/particle/momentum/class_4/assigned/samples": 82514.0, "metrics/particle/momentum/class_4/efficiency": 0.43702244758605957, "metrics/particle/momentum/class_4/event/energy_bias": 215.68240356445312, "metrics/particle/momentum/class_4/event/p3_error_mean": 338.426513671875, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 11.216645240783691, "metrics/particle/momentum/class_4/event/px_bias": 2.0280649662017822, "metrics/particle/momentum/class_4/event/py_bias": 1.1811721324920654, "metrics/particle/momentum/class_4/event/pz_abs_mean": 337.7927551269531, "metrics/particle/momentum/class_4/event/pz_bias": 27.93695831298828, "metrics/particle/momentum/class_4/event/pz_residual_rms": 497.1452331542969, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.05124005675315857, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.25068342685699463, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21658672392368317, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.042297590523958206, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2569938898086548, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02279384806752205, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2846431732177734, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.06163027510046959, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.960623264312744, "metrics/particle/momentum/class_5/accepted/samples": 10924.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.422938585281372, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.0560379028320312, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.987090528011322, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6923022270202637, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.38521334528923035, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.246161937713623, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.14900673925876617, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.428325176239014, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.15148241817951202, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 17.584503173828125, "metrics/particle/momentum/class_5/assigned/samples": 28945.0, "metrics/particle/momentum/class_5/efficiency": 0.3773793578147888, "metrics/particle/momentum/class_5/event/energy_bias": 19.26134490966797, "metrics/particle/momentum/class_5/event/p3_error_mean": 117.395263671875, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 26.92400550842285, "metrics/particle/momentum/class_5/event/px_bias": 3.133341073989868, "metrics/particle/momentum/class_5/event/py_bias": 2.266082286834717, "metrics/particle/momentum/class_5/event/pz_abs_mean": 108.99018096923828, "metrics/particle/momentum/class_5/event/pz_bias": 15.72319507598877, "metrics/particle/momentum/class_5/event/pz_residual_rms": 173.59210205078125, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.01899784617125988, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.09455939382314682, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.09418705105781555, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.049116190522909164, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.13794301450252533, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.013916500844061375, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.7089099884033203, "metrics/particle/momentum/class_6/accepted/pz_bias": 0.04658704996109009, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 2.595914125442505, "metrics/particle/momentum/class_6/accepted/samples": 503.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.27816033363342285, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 0.9932729601860046, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.6079457402229309, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.4538254141807556, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.09391304105520248, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 3.461914300918579, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.04521739110350609, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 2.15895676612854, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.06401477009057999, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 8.975604057312012, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8747826218605042, "metrics/particle/momentum/class_6/event/energy_bias": -9.142346382141113, "metrics/particle/momentum/class_6/event/p3_error_mean": 7.731316089630127, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 3.071476459503174, "metrics/particle/momentum/class_6/event/px_bias": -0.5420016050338745, "metrics/particle/momentum/class_6/event/py_bias": -0.5693544149398804, "metrics/particle/momentum/class_6/event/pz_abs_mean": 6.656785011291504, "metrics/particle/momentum/class_6/event/pz_bias": -0.6659039855003357, "metrics/particle/momentum/class_6/event/pz_residual_rms": 18.445463180541992, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.014170133508741856, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.051153261214494705, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.041627105325460434, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.011444038711488247, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.0753943920135498, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.015625, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.8791165947914124, "metrics/particle/momentum/class_7/accepted/pz_bias": 0.13912838697433472, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 3.78179931640625, "metrics/particle/momentum/class_7/accepted/samples": 192.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.04968969151377678, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.08863720297813416, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.061415184289216995, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.0260878037661314, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.02512562833726406, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.1595108062028885, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.015075377188622952, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.9095929861068726, "metrics/particle/momentum/class_7/assigned/pz_bias": 0.1766384243965149, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 3.7451844215393066, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9648241400718689, "metrics/particle/momentum/class_7/event/energy_bias": -1.6288015842437744, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.4807212352752686, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 1.0967122316360474, "metrics/particle/momentum/class_7/event/px_bias": 0.011789090931415558, "metrics/particle/momentum/class_7/event/py_bias": -0.062144935131073, "metrics/particle/momentum/class_7/event/pz_abs_mean": 2.0235893726348877, "metrics/particle/momentum/class_7/event/pz_bias": 0.0038493871688842773, "metrics/particle/momentum/class_7/event/pz_residual_rms": 5.5588579177856445, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 3.2689406871795654, "step_valid_time": 10277.251968383789, "step_total_time": 10280.520909309387, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.063634516162524, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3739784781496046, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84784, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16653, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13254, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.20439398588524088, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19641677675033026, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.20032599738961498, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8035832232496698, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1626756673826327, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15632666540856766, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15943798531207334, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.20439398588524088, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.059265473859678, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.39570297959502865, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84526, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15704, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12340, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.1933061706815692, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18578898800369117, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18947304919615118, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8142110119963089, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1518974876598678, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1459905827792632, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14888547039483607, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.1933061706815692, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0614272632055275, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.41679452870916045, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 83981, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 14978, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11617, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18539652675488, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.17834986485038282, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.1818049402197002, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8216501351496172, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1437943284358019, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1383289077291292, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14100867876433817, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18539652675488} \ No newline at end of file diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_60000.json b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_60000.json new file mode 100644 index 0000000000000000000000000000000000000000..8430a6ca3809d8cb29e648d8dda2efedcfcdecde --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_60000.json @@ -0,0 +1 @@ +{"epoch": 0.2585560506425118, "train": {"Classification_binary": 3.4145634174346924, "Classification": 0.015435723587870598, "Regression_pt": 0.5495187044143677, "Regression_eta": 0.07161962985992432, "Regression_sin_phi": 0.04267934709787369, "Regression_cos_phi": 0.044938042759895325, "Regression_energy": 0.5841283798217773, "Total": 14.380983352661133, "Total_unweighted": 4.722883244976401}, "train_group": {"physical_pu": {"Classification_binary": 2.938253457729633, "Classification": 0.0156483648365343, "Regression_pt": 0.23108267669494337, "Regression_eta": 0.10541662295441051, "Regression_sin_phi": 0.04734203027008654, "Regression_cos_phi": 0.047322247054550674, "Regression_energy": 0.27536231999868876, "Total": 10.765070234026227, "Total_unweighted": 3.660427719538847}, "physical_nopu": {"Classification_binary": 3.500585763499026, "Classification": 0.014542548602694856, "Regression_pt": 0.3799812380958049, "Regression_eta": 0.06996222298973465, "Regression_sin_phi": 0.04762795515465002, "Regression_cos_phi": 0.04793987784645345, "Regression_energy": 0.42587664766017885, "Total": 12.73168591959904, "Total_unweighted": 4.486516253848543}}, "valid": {"Classification_binary": 3.2186943689982095, "Classification": 0.01510017861922582, "Regression_pt": 0.3057291905085246, "Regression_eta": 0.08825242519378662, "Regression_sin_phi": 0.04779812196890513, "Regression_cos_phi": 0.04770760734875997, "Regression_energy": 0.35026009877522785, "Total": 11.7671267191569, "group/physical_pu/Classification": 0.015580580880244574, "group/physical_pu/Classification_binary": 2.936125119527181, "group/physical_pu/Regression_cos_phi": 0.047434642910957336, "group/physical_pu/Regression_energy": 0.2753988901774089, "group/physical_pu/Regression_eta": 0.10514133175214131, "group/physical_pu/Regression_pt": 0.23113477230072021, "group/physical_pu/Regression_sin_phi": 0.04739505549271902, "group/physical_pu/Total": 10.764801025390625, "group/physical_pu/Total_unweighted": 3.6582103930413723, "group/physical_nopu/Classification": 0.014619776358207067, "group/physical_nopu/Classification_binary": 3.5012636184692383, "group/physical_nopu/Regression_cos_phi": 0.04798058172067007, "group/physical_nopu/Regression_energy": 0.425121267636617, "group/physical_nopu/Regression_eta": 0.07136351863543193, "group/physical_nopu/Regression_pt": 0.3803236087163289, "group/physical_nopu/Regression_sin_phi": 0.04820119341214498, "group/physical_nopu/Total": 12.769449869791666, "group/physical_nopu/Total_unweighted": 4.488873564948638, "Total_unweighted": 4.07354199141264, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.014875072985887527, "diagnostic/loss/cms_pf/Classification_binary": 3.2110869884490967, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.04709682613611221, "diagnostic/loss/cms_pf/Regression_energy": 0.33812665939331055, "diagnostic/loss/cms_pf/Regression_eta": 0.08832606673240662, "diagnostic/loss/cms_pf/Regression_pt": 0.29411762952804565, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.04582229629158974, "diagnostic/regression/cms_pf/energy_pred_mean": 0.21577607095241547, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.35647737979888916, "diagnostic/regression/cms_pf/energy_residual_mean": 0.10500984638929367, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5856466889381409, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.11604896187782288, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.329573392868042, "diagnostic/regression/cms_pf/pt_residual_mean": 0.1268211305141449, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5482721328735352, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8888676762580872, "metrics/particle/class_1/pid_accuracy": 0.9784260988235474, "metrics/particle/class_2/efficiency": 0.2576635181903839, "metrics/particle/class_2/pid_accuracy": 0.20741140842437744, "metrics/particle/class_3/efficiency": 0.39246293902397156, "metrics/particle/class_3/pid_accuracy": 0.4775342643260956, "metrics/particle/class_4/efficiency": 0.4384652376174927, "metrics/particle/class_4/pid_accuracy": 0.6627237200737, "metrics/particle/class_5/efficiency": 0.37579357624053955, "metrics/particle/class_5/pid_accuracy": 0.6313654780387878, "metrics/particle/class_6/efficiency": 0.8609890937805176, "metrics/particle/class_6/pid_accuracy": 0.05793284997344017, "metrics/particle/class_7/efficiency": 0.9622716307640076, "metrics/particle/class_7/pid_accuracy": 0.2525794506072998, "metrics/particle/count/bias_mean": 65.85123443603516, "metrics/particle/count/mae": 87.54329681396484, "metrics/particle/count/prediction_mean": 1594.4547119140625, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.08422073721885681, "metrics/particle/event/energy_response_mean": 0.938349187374115, "metrics/particle/event/met_abs_error": 19.844881057739258, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04800865054130554, "metrics/particle/event/scalar_pt_response_mean": 1.0138803720474243, "metrics/particle/event/vector_pt_closure": 0.02892111800611019, "metrics/particle/matched/delta_eta_abs_mean": 0.019016768783330917, "metrics/particle/matched/delta_phi_abs_mean": 0.016367536038160324, "metrics/particle/matched/delta_r_mean": 0.02809520997107029, "metrics/particle/matched/energy_relative_abs_mean": 0.11695162206888199, "metrics/particle/matched/pid_accuracy": 0.8466914892196655, "metrics/particle/matched/pt_relative_abs_mean": 0.10529868304729462, "metrics/particle/matching/duplicate_fraction": 0.11078944802284241, "metrics/particle/matching/efficiency": 0.6358110308647156, "metrics/particle/matching/f1": 0.6224046349525452, "metrics/particle/matching/purity": 0.6095519661903381, "metrics/particle/matching/target_coverage_dr0p05": 0.5136681795120239, "metrics/particle/matching/target_coverage_dr0p10": 0.7009254693984985, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5566532015800476, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7539857625961304, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017545325681567192, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06777143478393555, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05798335373401642, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0037345937453210354, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.10922819375991821, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0019297963008284569, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.2236146628856659, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0008282208582386374, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4277029037475586, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68401.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.46856388449668884, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1626332998275757, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.39006543159484863, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.2646915018558502, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1380600780248642, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.4265379905700684, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.04010588303208351, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8832788467407227, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.09390333294868469, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.83739185333252, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87643.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7803789973258972, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 72.05775451660156, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 47.74361801147461, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 15.88571834564209, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.8925113677978516, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.4894787073135376, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 43.08974075317383, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -3.591346263885498, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 70.89399719238281, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.041026126593351364, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16132844984531403, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15879611670970917, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.017418744042515755, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21199481189250946, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17431314289569855, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.4940185546875, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.025060119107365608, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.616721153259277, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 120622.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5274620056152344, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9681833386421204, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9563128352165222, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.742109477519989, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.321540892124176, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3.859219789505005, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.37841489911079407, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.770702362060547, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.06947915256023407, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.253456115722656, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228593.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5214710831642151, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 77.34902954101562, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 383.45703125, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 19.708324432373047, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.5404535531997681, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.9086540937423706, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 381.7860412597656, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -49.68872833251953, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 561.2184448242188, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01369436178356409, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08322806656360626, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.05045046657323837, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.007902121171355247, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10809779912233353, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00068679015384987, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.07911860942840576, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 9.589409455657005e-05, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8520248532295227, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59698.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3341236710548401, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.6735548973083496, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.28371602296829224, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.18696148693561554, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10565535724163055, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.345895290374756, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.02253556251525879, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.095869779586792, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.06900013238191605, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.690876483917236, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72197.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.826865017414093, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 19.979312896728516, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 21.46718406677246, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 16.41044044494629, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 2.0208446979522705, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.6718355417251587, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 11.771590232849121, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.5910230875015259, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 24.38597297668457, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02800847962498665, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11685358732938766, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10506638139486313, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.007577941287308931, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16697649657726288, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08523204922676086, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.744915723800659, "metrics/particle/momentum/all/accepted/pz_bias": 0.011948630213737488, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.135141372680664, "metrics/particle/momentum/all/accepted/samples": 248721.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0667378902435303, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1431629657745361, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7035354375839233, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5312047600746155, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2400156557559967, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 4.192656517028809, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.23593515157699585, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.909680366516113, "metrics/particle/momentum/all/assigned/pz_bias": -0.07490099221467972, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.34491539001465, "metrics/particle/momentum/all/assigned/samples": 388433.0, "metrics/particle/momentum/all/efficiency": 0.6358548998832703, "metrics/particle/momentum/all/event/energy_bias": 169.38609313964844, "metrics/particle/momentum/all/event/p3_error_mean": 384.5906066894531, "metrics/particle/momentum/all/event/pt_vector_error_mean": 27.249832153320312, "metrics/particle/momentum/all/event/px_bias": 4.4538092613220215, "metrics/particle/momentum/all/event/py_bias": -1.0910109281539917, "metrics/particle/momentum/all/event/pz_abs_mean": 381.2487487792969, "metrics/particle/momentum/all/event/pz_bias": -54.871097564697266, "metrics/particle/momentum/all/event/pz_residual_rms": 559.8775634765625, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.014420065097510815, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06366710364818573, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05176354572176933, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0027514081448316574, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0959731712937355, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012302777031436563, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.20596368610858917, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0020271658431738615, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.128072738647461, "metrics/particle/momentum/class_1/accepted/samples": 171506.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.22586502134799957, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.0176490545272827, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.32632729411125183, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2648742198944092, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07744257897138596, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.740009784698486, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.02312903292477131, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1625995635986328, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.006956970784813166, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.846756935119629, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8896093368530273, "metrics/particle/momentum/class_1/event/energy_bias": 30.637096405029297, "metrics/particle/momentum/class_1/event/p3_error_mean": 56.34319305419922, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 17.266414642333984, "metrics/particle/momentum/class_1/event/px_bias": 1.3511649370193481, "metrics/particle/momentum/class_1/event/py_bias": -1.827139973640442, "metrics/particle/momentum/class_1/event/pz_abs_mean": 50.082645416259766, "metrics/particle/momentum/class_1/event/pz_bias": -6.946652412414551, "metrics/particle/momentum/class_1/event/pz_residual_rms": 79.4105224609375, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.058326829224824905, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23852266371250153, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23370914161205292, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07485488802194595, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27344799041748047, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.11232740432024002, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.2240095138549805, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.14501582086086273, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 10.199861526489258, "metrics/particle/momentum/class_2/accepted/samples": 8039.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.1887292861938477, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.5207455158233643, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1339322328567505, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6475456357002258, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.47228124737739563, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.261968612670898, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.35049256682395935, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.575319290161133, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.03825611248612404, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 34.80039596557617, "metrics/particle/momentum/class_2/assigned/samples": 31062.0, "metrics/particle/momentum/class_2/efficiency": 0.2587716579437256, "metrics/particle/momentum/class_2/event/energy_bias": -290.7675476074219, "metrics/particle/momentum/class_2/event/p3_error_mean": 170.98614501953125, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 28.135108947753906, "metrics/particle/momentum/class_2/event/px_bias": 1.5395337343215942, "metrics/particle/momentum/class_2/event/py_bias": -0.013355791568756104, "metrics/particle/momentum/class_2/event/pz_abs_mean": 165.20492553710938, "metrics/particle/momentum/class_2/event/pz_bias": -31.338390350341797, "metrics/particle/momentum/class_2/event/pz_residual_rms": 231.96388244628906, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.06069231405854225, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.226774662733078, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.22339510917663574, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.006688535679131746, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.26271912455558777, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.31998082995414734, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.428594589233398, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.00327088194899261, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 14.04508113861084, "metrics/particle/momentum/class_3/accepted/samples": 20870.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.05039381980896, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.9654829502105713, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.1098041534423828, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.9346626400947571, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.42296332120895386, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 3.735515832901001, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5400285720825195, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.479318618774414, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.030153002589941025, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.56371307373047, "metrics/particle/momentum/class_3/assigned/samples": 52475.0, "metrics/particle/momentum/class_3/efficiency": 0.39154988527297974, "metrics/particle/momentum/class_3/event/energy_bias": 555.7579956054688, "metrics/particle/momentum/class_3/event/p3_error_mean": 287.1891174316406, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.75274658203125, "metrics/particle/momentum/class_3/event/px_bias": -1.309521198272705, "metrics/particle/momentum/class_3/event/py_bias": -0.5149406790733337, "metrics/particle/momentum/class_3/event/pz_abs_mean": 286.6979675292969, "metrics/particle/momentum/class_3/event/pz_bias": 20.593626022338867, "metrics/particle/momentum/class_3/event/pz_residual_rms": 455.6375427246094, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.059665512293577194, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.22956065833568573, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22630125284194946, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.05537356063723564, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.26555749773979187, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.35765668749809265, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 10.951128959655762, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.06802381575107574, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.3358154296875, "metrics/particle/momentum/class_4/accepted/samples": 36700.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.8644541501998901, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0648529529571533, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.072549819946289, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8045613765716553, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3682715892791748, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.1243979930877686, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5301299691200256, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.420154571533203, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.22840051352977753, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 40.002342224121094, "metrics/particle/momentum/class_4/assigned/samples": 82393.0, "metrics/particle/momentum/class_4/efficiency": 0.4354325830936432, "metrics/particle/momentum/class_4/event/energy_bias": -126.92671203613281, "metrics/particle/momentum/class_4/event/p3_error_mean": 328.99761962890625, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 11.238605499267578, "metrics/particle/momentum/class_4/event/px_bias": 1.3754750490188599, "metrics/particle/momentum/class_4/event/py_bias": -1.5178277492523193, "metrics/particle/momentum/class_4/event/pz_abs_mean": 328.31842041015625, "metrics/particle/momentum/class_4/event/pz_bias": -33.12202072143555, "metrics/particle/momentum/class_4/event/pz_residual_rms": 492.051025390625, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0509210005402565, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2760161757469177, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2156410962343216, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03994281217455864, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.25590798258781433, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.024734335020184517, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.301884412765503, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.030967745929956436, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.7818803787231445, "metrics/particle/momentum/class_5/accepted/samples": 10916.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4321320056915283, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.124756336212158, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9740903973579407, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6762741208076477, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.38105109333992004, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.6741158962249756, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.1470232605934143, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.458060264587402, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.2089371383190155, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 17.637203216552734, "metrics/particle/momentum/class_5/assigned/samples": 28941.0, "metrics/particle/momentum/class_5/efficiency": 0.37710297107696533, "metrics/particle/momentum/class_5/event/energy_bias": 10.694421768188477, "metrics/particle/momentum/class_5/event/p3_error_mean": 115.06159973144531, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 26.19489860534668, "metrics/particle/momentum/class_5/event/px_bias": 1.9993643760681152, "metrics/particle/momentum/class_5/event/py_bias": 3.323051929473877, "metrics/particle/momentum/class_5/event/pz_abs_mean": 106.91740417480469, "metrics/particle/momentum/class_5/event/pz_bias": -2.487147092819214, "metrics/particle/momentum/class_5/event/pz_residual_rms": 171.19320678710938, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.019397146999835968, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.0951509177684784, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.09386449307203293, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.042668282985687256, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.13732090592384338, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.016032064333558083, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.7845061421394348, "metrics/particle/momentum/class_6/accepted/pz_bias": 0.14558306336402893, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 3.9167709350585938, "metrics/particle/momentum/class_6/accepted/samples": 499.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.2759090065956116, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 1.2315930128097534, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.5772098898887634, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.42675331234931946, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.10260869562625885, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 2.9673523902893066, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.04173913225531578, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 2.2059364318847656, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.19341152906417847, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 10.524917602539062, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8678261041641235, "metrics/particle/momentum/class_6/event/energy_bias": -8.244858741760254, "metrics/particle/momentum/class_6/event/p3_error_mean": 8.342958450317383, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 3.16432523727417, "metrics/particle/momentum/class_6/event/px_bias": -0.4247777462005615, "metrics/particle/momentum/class_6/event/py_bias": -0.4320392310619354, "metrics/particle/momentum/class_6/event/pz_abs_mean": 7.241119384765625, "metrics/particle/momentum/class_6/event/pz_bias": -1.4489028453826904, "metrics/particle/momentum/class_6/event/pz_residual_rms": 21.2894229888916, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.012429735623300076, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.047291070222854614, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.03864800184965134, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.011296870186924934, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.06979361176490784, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.015706805512309074, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.8027216196060181, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.047375865280628204, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 3.4263713359832764, "metrics/particle/momentum/class_7/accepted/samples": 191.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.03744525462388992, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.06744683533906937, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.05797555297613144, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.023803487420082092, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.02512562833726406, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.1411324441432953, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.015075377188622952, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.8144238591194153, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.02788110263645649, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 3.376054286956787, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9597989916801453, "metrics/particle/momentum/class_7/event/energy_bias": -1.764326810836792, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.383479356765747, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 1.0298497676849365, "metrics/particle/momentum/class_7/event/px_bias": -0.07742969691753387, "metrics/particle/momentum/class_7/event/py_bias": -0.10875997692346573, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.9731884002685547, "metrics/particle/momentum/class_7/event/pz_bias": -0.12160628288984299, "metrics/particle/momentum/class_7/event/pz_residual_rms": 5.291667461395264, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 3.147183656692505, "step_valid_time": 12418.47379040718, "step_total_time": 12421.62097454071, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.055397011190048, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.36360516860694114, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84470, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16811, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13581, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.2063332310524701, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.199017402628152, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.20260929826147217, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.800982597371848, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.16668916845658177, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.16077897478394695, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.16368073759378107, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.2063332310524701, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.055021245334093, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.38694517124744165, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84166, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15804, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12545, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19453710656211917, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.187771784330965, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.1910945860161422, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.812228215669035, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15442090621499527, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14905068554998455, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15168828028173273, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19453710656211917, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0554804164851475, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.39606281980377955, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 83797, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15072, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11990, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.1865600514921586, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.17986324092747952, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.1831504502205534, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8201367590725205, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14841129361670524, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14308388128453287, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.1456989051316637, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.1865600514921586} \ No newline at end of file diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_70000.json b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_70000.json new file mode 100644 index 0000000000000000000000000000000000000000..952977baa2d86c1485ce1b8211fd7f16224b8341 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_70000.json @@ -0,0 +1 @@ +{"epoch": 0.30164872574959706, "train": {"Classification_binary": 3.3975234031677246, "Classification": 0.01641874574124813, "Regression_pt": 0.5834938883781433, "Regression_eta": 0.06378737837076187, "Regression_sin_phi": 0.044774655252695084, "Regression_cos_phi": 0.048739030957221985, "Regression_energy": 0.6182875037193298, "Total": 14.915570259094238, "Total_unweighted": 4.773024605587125}, "train_group": {"physical_pu": {"Classification_binary": 2.92859417789585, "Classification": 0.015651008574762842, "Regression_pt": 0.23049615994914546, "Regression_eta": 0.10502093948505736, "Regression_sin_phi": 0.04724594668208898, "Regression_cos_phi": 0.04716474890381425, "Regression_energy": 0.27458331152633, "Total": 10.73415022629958, "Total_unweighted": 3.648756293017049}, "physical_nopu": {"Classification_binary": 3.485715519345895, "Classification": 0.014484312714203553, "Regression_pt": 0.37415658499803744, "Regression_eta": 0.06924225752240885, "Regression_sin_phi": 0.04727594109766423, "Regression_cos_phi": 0.047743842727018, "Regression_energy": 0.4197780010592688, "Total": 12.617375323481292, "Total_unweighted": 4.458396459464496}}, "valid": {"Classification_binary": 3.206186294555664, "Classification": 0.015039488673210144, "Regression_pt": 0.30412240823109943, "Regression_eta": 0.08795579274495442, "Regression_sin_phi": 0.047652999560038246, "Regression_cos_phi": 0.047643120090166725, "Regression_energy": 0.34882819652557373, "Total": 11.724302927652994, "group/physical_pu/Classification": 0.015555821359157562, "group/physical_pu/Classification_binary": 2.9272677103678384, "group/physical_pu/Regression_cos_phi": 0.04741378625233968, "group/physical_pu/Regression_energy": 0.2745494643847148, "group/physical_pu/Regression_eta": 0.10500325759251912, "group/physical_pu/Regression_pt": 0.23027708133061728, "group/physical_pu/Regression_sin_phi": 0.04735705256462097, "group/physical_pu/Total": 10.741582234700521, "group/physical_pu/Total_unweighted": 3.6474241738518076, "group/physical_nopu/Classification": 0.014523155987262726, "group/physical_nopu/Classification_binary": 3.485104242960612, "group/physical_nopu/Regression_cos_phi": 0.04787244896094004, "group/physical_nopu/Regression_energy": 0.42310698827107746, "group/physical_nopu/Regression_eta": 0.07090833286444347, "group/physical_nopu/Regression_pt": 0.3779677947362264, "group/physical_nopu/Regression_sin_phi": 0.04794895648956299, "group/physical_nopu/Total": 12.707021077473959, "group/physical_nopu/Total_unweighted": 4.4674319202701245, "Total_unweighted": 4.057428300380707, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.01480147149413824, "diagnostic/loss/cms_pf/Classification_binary": 3.1990859508514404, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.04714665189385414, "diagnostic/loss/cms_pf/Regression_energy": 0.33628177642822266, "diagnostic/loss/cms_pf/Regression_eta": 0.08809641748666763, "diagnostic/loss/cms_pf/Regression_pt": 0.29142889380455017, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.045801158994436264, "diagnostic/regression/cms_pf/energy_pred_mean": 0.21886709332466125, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.35644716024398804, "diagnostic/regression/cms_pf/energy_residual_mean": 0.10810086876153946, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5844106078147888, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.11505264043807983, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.327809602022171, "diagnostic/regression/cms_pf/pt_residual_mean": 0.12582482397556305, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5452785491943359, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8894505500793457, "metrics/particle/class_1/pid_accuracy": 0.9782962799072266, "metrics/particle/class_2/efficiency": 0.25800377130508423, "metrics/particle/class_2/pid_accuracy": 0.20517532527446747, "metrics/particle/class_3/efficiency": 0.39456477761268616, "metrics/particle/class_3/pid_accuracy": 0.47532927989959717, "metrics/particle/class_4/efficiency": 0.4381714165210724, "metrics/particle/class_4/pid_accuracy": 0.6651012897491455, "metrics/particle/class_5/efficiency": 0.3785693049430847, "metrics/particle/class_5/pid_accuracy": 0.6348116397857666, "metrics/particle/class_6/efficiency": 0.864389955997467, "metrics/particle/class_6/pid_accuracy": 0.053606558591127396, "metrics/particle/class_7/efficiency": 0.9630659222602844, "metrics/particle/class_7/pid_accuracy": 0.2453608214855194, "metrics/particle/count/bias_mean": 67.20149993896484, "metrics/particle/count/mae": 88.87467193603516, "metrics/particle/count/prediction_mean": 1595.8050537109375, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.08166882395744324, "metrics/particle/event/energy_response_mean": 0.9377023577690125, "metrics/particle/event/met_abs_error": 18.99333381652832, "metrics/particle/event/scalar_pt_relative_abs_error": 0.048069849610328674, "metrics/particle/event/scalar_pt_response_mean": 1.0080896615982056, "metrics/particle/event/vector_pt_closure": 0.02783220261335373, "metrics/particle/matched/delta_eta_abs_mean": 0.019087817519903183, "metrics/particle/matched/delta_phi_abs_mean": 0.016109442338347435, "metrics/particle/matched/delta_r_mean": 0.028004219755530357, "metrics/particle/matched/energy_relative_abs_mean": 0.11840443313121796, "metrics/particle/matched/pid_accuracy": 0.8466405272483826, "metrics/particle/matched/pt_relative_abs_mean": 0.1049387976527214, "metrics/particle/matching/duplicate_fraction": 0.11117812991142273, "metrics/particle/matching/efficiency": 0.6365563869476318, "metrics/particle/matching/f1": 0.6228649616241455, "metrics/particle/matching/purity": 0.6097501516342163, "metrics/particle/matching/target_coverage_dr0p05": 0.5143890380859375, "metrics/particle/matching/target_coverage_dr0p10": 0.701654851436615, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5570228695869446, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7548586130142212, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017473360523581505, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06821375340223312, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05651235207915306, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.00189311895519495, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.10776578634977341, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0018539875745773315, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.21714048087596893, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.010386552661657333, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2983543872833252, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68501.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.46707969903945923, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1537140607833862, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.3788776695728302, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.25718382000923157, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13839922845363617, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.8230226039886475, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.04024188593029976, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8409209251403809, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.06546512246131897, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.368584632873535, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87645.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.781519889831543, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 64.42893981933594, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 45.74162292480469, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 15.502214431762695, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.5537347793579102, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.35560494661331177, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 41.20674514770508, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 3.004528045654297, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 69.36094665527344, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.040886130183935165, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1618909239768982, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15905706584453583, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.013375124894082546, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21254950761795044, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17410363256931305, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.497255802154541, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.002667757449671626, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.652054786682129, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 120905.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5217465162277222, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9548666477203369, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.954358696937561, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7398301959037781, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3209260404109955, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 6.3265790939331055, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.37760302424430847, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.741141319274902, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.08024302870035172, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.23883056640625, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228629.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5226945281028748, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.475695610046387, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 382.31109619140625, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 19.34082794189453, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.48691341280937195, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.10182885825634003, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 380.72100830078125, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -69.76367950439453, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 566.192138671875, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013732078485190868, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0866931602358818, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.049779850989580154, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011921069584786892, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10679654031991959, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007207629969343543, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.07897929847240448, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0006627143011428416, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.77913898229599, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59659.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3327207565307617, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.27367103099823, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.2567446827888489, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.16271013021469116, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10519827902317047, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 2.1680572032928467, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.022023076191544533, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0657154321670532, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.07404550164937973, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.5637993812561035, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72197.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8263248205184937, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 18.953895568847656, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 20.316072463989258, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 15.023720741271973, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.7038979530334473, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.70211124420166, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 11.531972885131836, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.15777406096458435, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 21.814804077148438, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02794259786605835, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11811438947916031, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10467854887247086, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.0031166155822575092, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1669033318758011, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08519864082336426, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.747201919555664, "metrics/particle/momentum/all/accepted/pz_bias": 0.003992925398051739, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.155783653259277, "metrics/particle/momentum/all/accepted/samples": 249065.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0628180503845215, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0589791536331177, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6948704123497009, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.523680567741394, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2396523803472519, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 5.121337890625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.23540495336055756, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.87762451171875, "metrics/particle/momentum/all/assigned/pz_bias": -0.07575708627700806, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.28832244873047, "metrics/particle/momentum/all/assigned/samples": 388471.0, "metrics/particle/momentum/all/efficiency": 0.6367343068122864, "metrics/particle/momentum/all/event/energy_bias": 74.90713500976562, "metrics/particle/momentum/all/event/p3_error_mean": 383.5042419433594, "metrics/particle/momentum/all/event/pt_vector_error_mean": 26.170520782470703, "metrics/particle/momentum/all/event/px_bias": 0.6370766162872314, "metrics/particle/momentum/all/event/py_bias": -3.1595449447631836, "metrics/particle/momentum/all/event/pz_abs_mean": 380.1885070800781, "metrics/particle/momentum/all/event/pz_bias": -66.60137939453125, "metrics/particle/momentum/all/event/pz_residual_rms": 564.0942993164062, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.014245293103158474, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06530918180942535, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05096229910850525, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.010031212121248245, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0953029990196228, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012008300982415676, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.2038096934556961, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0022020279429852962, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.0751105546951294, "metrics/particle/momentum/class_1/accepted/samples": 171548.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.22102563083171844, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.8675195574760437, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.32677555084228516, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.27241283655166626, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07748407870531082, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.384988307952881, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.022599954158067703, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1424692869186401, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.006556566338986158, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.674169540405273, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8898271918296814, "metrics/particle/momentum/class_1/event/energy_bias": 86.70579528808594, "metrics/particle/momentum/class_1/event/p3_error_mean": 56.525455474853516, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 16.811731338500977, "metrics/particle/momentum/class_1/event/px_bias": -1.6791901588439941, "metrics/particle/momentum/class_1/event/py_bias": -1.602469801902771, "metrics/particle/momentum/class_1/event/pz_abs_mean": 50.73290252685547, "metrics/particle/momentum/class_1/event/pz_bias": 1.244741439819336, "metrics/particle/momentum/class_1/event/pz_residual_rms": 83.14356994628906, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.05855856463313103, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23817557096481323, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23432782292366028, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07275865972042084, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27433931827545166, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.1095079556107521, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.18751335144043, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.17652711272239685, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 10.114176750183105, "metrics/particle/momentum/class_2/accepted/samples": 8109.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.1769931316375732, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.4667878150939941, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.126421570777893, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6412333846092224, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4705844521522522, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.347206115722656, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.35317984223365784, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.55543327331543, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.1722560077905655, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 34.72509765625, "metrics/particle/momentum/class_2/assigned/samples": 31055.0, "metrics/particle/momentum/class_2/efficiency": 0.26102492213249207, "metrics/particle/momentum/class_2/event/energy_bias": -327.7789001464844, "metrics/particle/momentum/class_2/event/p3_error_mean": 170.91734313964844, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 28.27008056640625, "metrics/particle/momentum/class_2/event/px_bias": 0.0625876933336258, "metrics/particle/momentum/class_2/event/py_bias": -3.6848561763763428, "metrics/particle/momentum/class_2/event/pz_abs_mean": 165.08575439453125, "metrics/particle/momentum/class_2/event/pz_bias": 1.8030800819396973, "metrics/particle/momentum/class_2/event/pz_residual_rms": 229.1478729248047, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.06072705611586571, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.22712205350399017, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.22377797961235046, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.007439173758029938, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.26313501596450806, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.3155420422554016, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.383814811706543, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.03467449173331261, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 13.988295555114746, "metrics/particle/momentum/class_3/accepted/samples": 21059.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.0371413230895996, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.946092426776886, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.0724173784255981, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.8929338455200195, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4185979962348938, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 2.5721027851104736, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5365175008773804, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.355623245239258, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.08650267869234085, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.52417755126953, "metrics/particle/momentum/class_3/assigned/samples": 52468.0, "metrics/particle/momentum/class_3/efficiency": 0.3950957655906677, "metrics/particle/momentum/class_3/event/energy_bias": 491.6355285644531, "metrics/particle/momentum/class_3/event/p3_error_mean": 285.2580871582031, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.64515495300293, "metrics/particle/momentum/class_3/event/px_bias": 0.9073807001113892, "metrics/particle/momentum/class_3/event/py_bias": -0.5319738984107971, "metrics/particle/momentum/class_3/event/pz_abs_mean": 284.8226013183594, "metrics/particle/momentum/class_3/event/pz_bias": 11.667356491088867, "metrics/particle/momentum/class_3/event/pz_residual_rms": 454.6720886230469, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.05978304147720337, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.2300337851047516, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22711993753910065, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.058164872229099274, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2662966549396515, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.3600621521472931, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 10.991436004638672, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.005756518803536892, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.442508697509766, "metrics/particle/momentum/class_4/accepted/samples": 36677.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.8675216436386108, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0641638040542603, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0578138828277588, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.782675564289093, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.36957207322120667, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.246259927749634, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.529826283454895, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.428434371948242, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.34501346945762634, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 40.003597259521484, "metrics/particle/momentum/class_4/assigned/samples": 82444.0, "metrics/particle/momentum/class_4/efficiency": 0.43515971302986145, "metrics/particle/momentum/class_4/event/energy_bias": -163.153076171875, "metrics/particle/momentum/class_4/event/p3_error_mean": 332.44134521484375, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 11.123727798461914, "metrics/particle/momentum/class_4/event/px_bias": 0.7969943284988403, "metrics/particle/momentum/class_4/event/py_bias": 1.3154171705245972, "metrics/particle/momentum/class_4/event/pz_abs_mean": 331.84942626953125, "metrics/particle/momentum/class_4/event/pz_bias": -82.529541015625, "metrics/particle/momentum/class_4/event/pz_residual_rms": 497.7835388183594, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.050782460719347, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2737170159816742, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2124752402305603, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03638383746147156, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2525465488433838, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.024212634190917015, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2849557399749756, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.02579309232532978, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.830203056335449, "metrics/particle/momentum/class_5/accepted/samples": 10986.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4386982917785645, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.097388744354248, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9728611707687378, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6700120568275452, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.38159075379371643, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.249337196350098, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.14753645658493042, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.373875141143799, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.10153047740459442, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 17.321657180786133, "metrics/particle/momentum/class_5/assigned/samples": 28942.0, "metrics/particle/momentum/class_5/efficiency": 0.3795211911201477, "metrics/particle/momentum/class_5/event/energy_bias": -2.651317596435547, "metrics/particle/momentum/class_5/event/p3_error_mean": 112.8171615600586, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 24.87757682800293, "metrics/particle/momentum/class_5/event/px_bias": 0.985776960849762, "metrics/particle/momentum/class_5/event/py_bias": 1.7971453666687012, "metrics/particle/momentum/class_5/event/pz_abs_mean": 105.63414001464844, "metrics/particle/momentum/class_5/event/pz_bias": 2.945955276489258, "metrics/particle/momentum/class_5/event/pz_residual_rms": 166.99464416503906, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.018189840018749237, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.09227627515792847, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.08975644409656525, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.03997870534658432, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.13369545340538025, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.014198782853782177, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.6612593531608582, "metrics/particle/momentum/class_6/accepted/pz_bias": -0.017417943105101585, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 2.3340322971343994, "metrics/particle/momentum/class_6/accepted/samples": 493.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.2799195349216461, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 0.8598246574401855, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.5406655669212341, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.3979315459728241, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.11478260904550552, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 2.5858871936798096, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.03478260710835457, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 1.869397521018982, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.21356625854969025, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 8.334346771240234, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8573912978172302, "metrics/particle/momentum/class_6/event/energy_bias": -8.212749481201172, "metrics/particle/momentum/class_6/event/p3_error_mean": 7.847841739654541, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 2.9327800273895264, "metrics/particle/momentum/class_6/event/px_bias": -0.3532017469406128, "metrics/particle/momentum/class_6/event/py_bias": -0.3182483911514282, "metrics/particle/momentum/class_6/event/pz_abs_mean": 6.8237714767456055, "metrics/particle/momentum/class_6/event/pz_bias": -1.5772236585617065, "metrics/particle/momentum/class_6/event/pz_residual_rms": 20.041948318481445, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.013172229751944542, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.05530690401792526, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.0414288230240345, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.01500976923853159, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.07853389531373978, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.010362694039940834, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.6371437907218933, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.019100159406661987, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 2.1424777507781982, "metrics/particle/momentum/class_7/accepted/samples": 193.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.029990261420607567, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.06888836622238159, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.061032410711050034, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.028256231918931007, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.02010050229728222, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.16996802389621735, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.01005025114864111, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.6570996046066284, "metrics/particle/momentum/class_7/assigned/pz_bias": 0.006739472504705191, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 2.1406421661376953, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9698492288589478, "metrics/particle/momentum/class_7/event/energy_bias": -1.6381676197052002, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.2782063484191895, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 0.9995932579040527, "metrics/particle/momentum/class_7/event/px_bias": -0.08327100425958633, "metrics/particle/momentum/class_7/event/py_bias": -0.13455957174301147, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.8786641359329224, "metrics/particle/momentum/class_7/event/pz_bias": -0.15574489533901215, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.81432580947876, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 3.0389463901519775, "step_valid_time": 14569.413124322891, "step_total_time": 14572.45207118988, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0519375502213462, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.35632031598160263, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84761, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16949, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13726, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.20802700214789813, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19996224678802751, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.20391491614331433, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8000377532119725, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.16846885547714022, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1619376836044879, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.16513871844847086, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.20802700214789813, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0543525575664663, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3806969524587711, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84520, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15895, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12714, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19565725821341967, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.1880619971604354, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19178445815913464, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8119380028395646, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15650118785312472, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1504259346900142, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15340343510759596, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19565725821341967, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0541179424102274, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3868301815755655, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 83888, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15335, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12263, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18981544517199123, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.1828032614915125, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.1862433733915483, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8171967385084875, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15179046652390796, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14618300591264544, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14893397377897338, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18981544517199123} \ No newline at end of file diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_80000.json b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_80000.json new file mode 100644 index 0000000000000000000000000000000000000000..f1d1166357471d68606f629870002bb38ba8d922 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_80000.json @@ -0,0 +1 @@ +{"epoch": 0.34474140085668237, "train": {"Classification_binary": 3.42079496383667, "Classification": 0.015797613188624382, "Regression_pt": 0.6092456579208374, "Regression_eta": 0.077316053211689, "Regression_sin_phi": 0.046377670019865036, "Regression_cos_phi": 0.04545070230960846, "Regression_energy": 0.6414522528648376, "Total": 15.306541442871094, "Total_unweighted": 4.856434913352132}, "train_group": {"physical_pu": {"Classification_binary": 2.920033077617268, "Classification": 0.015599548458479919, "Regression_pt": 0.2299636430465258, "Regression_eta": 0.10563151071687321, "Regression_sin_phi": 0.04717815393111208, "Regression_cos_phi": 0.04718988761305809, "Regression_energy": 0.2742203971841833, "Total": 10.724256651742119, "Total_unweighted": 3.6398162185675003}, "physical_nopu": {"Classification_binary": 3.4771474126410826, "Classification": 0.014398579425265555, "Regression_pt": 0.3695058732962582, "Regression_eta": 0.06913928891664291, "Regression_sin_phi": 0.047413186833049334, "Regression_cos_phi": 0.047643367565832255, "Regression_energy": 0.4153679373240707, "Total": 12.552305771310468, "Total_unweighted": 4.440615646002201}}, "valid": {"Classification_binary": 3.1997108459472656, "Classification": 0.015013723323742548, "Regression_pt": 0.30268073081970215, "Regression_eta": 0.08786755800247192, "Regression_sin_phi": 0.04761984944343567, "Regression_cos_phi": 0.04757488270600637, "Regression_energy": 0.3472586472829183, "Total": 11.694798787434896, "group/physical_pu/Classification": 0.015540855626265207, "group/physical_pu/Classification_binary": 2.921290397644043, "group/physical_pu/Regression_cos_phi": 0.04735963543256124, "group/physical_pu/Regression_energy": 0.27380265792210895, "group/physical_pu/Regression_eta": 0.1048716406027476, "group/physical_pu/Regression_pt": 0.22964815298716226, "group/physical_pu/Regression_sin_phi": 0.04732726514339447, "group/physical_pu/Total": 10.722736358642578, "group/physical_pu/Total_unweighted": 3.6398406053582826, "group/physical_nopu/Classification": 0.014486594746510187, "group/physical_nopu/Classification_binary": 3.4781322479248047, "group/physical_nopu/Regression_cos_phi": 0.047790125012397766, "group/physical_nopu/Regression_energy": 0.42071465651194256, "group/physical_nopu/Regression_eta": 0.07086347540219624, "group/physical_nopu/Regression_pt": 0.3757133483886719, "group/physical_nopu/Regression_sin_phi": 0.04791242380936941, "group/physical_nopu/Total": 12.666858673095703, "group/physical_nopu/Total_unweighted": 4.455612871795893, "Total_unweighted": 4.047726237525542, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.014792668633162975, "diagnostic/loss/cms_pf/Classification_binary": 3.1926515102386475, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.046970296651124954, "diagnostic/loss/cms_pf/Regression_energy": 0.3344350755214691, "diagnostic/loss/cms_pf/Regression_eta": 0.08800726383924484, "diagnostic/loss/cms_pf/Regression_pt": 0.289667546749115, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.045732785016298294, "diagnostic/regression/cms_pf/energy_pred_mean": 0.22280214726924896, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.35693851113319397, "diagnostic/regression/cms_pf/energy_residual_mean": 0.11203591525554657, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5855910778045654, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.11575590819120407, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.3279833197593689, "diagnostic/regression/cms_pf/pt_residual_mean": 0.1265280842781067, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5460577011108398, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8885336518287659, "metrics/particle/class_1/pid_accuracy": 0.9784116744995117, "metrics/particle/class_2/efficiency": 0.2570124864578247, "metrics/particle/class_2/pid_accuracy": 0.208572655916214, "metrics/particle/class_3/efficiency": 0.39325734972953796, "metrics/particle/class_3/pid_accuracy": 0.47853827476501465, "metrics/particle/class_4/efficiency": 0.4382863938808441, "metrics/particle/class_4/pid_accuracy": 0.662828266620636, "metrics/particle/class_5/efficiency": 0.37530115246772766, "metrics/particle/class_5/pid_accuracy": 0.6337324380874634, "metrics/particle/class_6/efficiency": 0.8594303727149963, "metrics/particle/class_6/pid_accuracy": 0.054080791771411896, "metrics/particle/class_7/efficiency": 0.9650516510009766, "metrics/particle/class_7/pid_accuracy": 0.2419753074645996, "metrics/particle/count/bias_mean": 62.82389450073242, "metrics/particle/count/mae": 85.6357421875, "metrics/particle/count/prediction_mean": 1591.4273681640625, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.08763309568166733, "metrics/particle/event/energy_response_mean": 0.9442132115364075, "metrics/particle/event/met_abs_error": 19.45379066467285, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04872076213359833, "metrics/particle/event/scalar_pt_response_mean": 1.005876064300537, "metrics/particle/event/vector_pt_closure": 0.028026586398482323, "metrics/particle/matched/delta_eta_abs_mean": 0.01898890547454357, "metrics/particle/matched/delta_phi_abs_mean": 0.016100313514471054, "metrics/particle/matched/delta_r_mean": 0.027912724763154984, "metrics/particle/matched/energy_relative_abs_mean": 0.11833887547254562, "metrics/particle/matched/pid_accuracy": 0.8468843102455139, "metrics/particle/matched/pt_relative_abs_mean": 0.1048445999622345, "metrics/particle/matching/duplicate_fraction": 0.11002369970083237, "metrics/particle/matching/efficiency": 0.635628342628479, "metrics/particle/matching/f1": 0.6228295564651489, "metrics/particle/matching/purity": 0.6105359792709351, "metrics/particle/matching/target_coverage_dr0p05": 0.5124279260635376, "metrics/particle/matching/target_coverage_dr0p10": 0.7001281976699829, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.557288646697998, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7539706230163574, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017514223232865334, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06948023289442062, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05778221786022186, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0032695012632757425, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1097029447555542, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.001958634704351425, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.21919061243534088, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.00021001711138524115, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2400165796279907, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68415.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4747454524040222, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.163819670677185, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.37705302238464355, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.24963705241680145, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13757416605949402, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.3556430339813232, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03985622897744179, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8775441646575928, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0934690460562706, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.575162887573242, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87640.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7805387377738953, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 58.461814880371094, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 43.54958724975586, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 15.627467155456543, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.5298576354980469, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -3.062798500061035, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 38.85554504394531, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.23950958251953125, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 60.063690185546875, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04060065746307373, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16112789511680603, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15831945836544037, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.015744373202323914, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2116205096244812, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17370444536209106, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.513372421264648, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.006588113494217396, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.681147575378418, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 120682.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.522275447845459, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9704249501228333, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9547302722930908, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.742287278175354, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.32223954796791077, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3.859234094619751, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.3785255253314972, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.750155448913574, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.013491056859493256, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.17546844482422, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228476.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5217304825782776, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 342.57916259765625, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 380.70166015625, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 19.474138259887695, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.29566752910614014, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -1.8001900911331177, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 378.5103759765625, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 34.0870246887207, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 553.4270629882812, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013841464184224606, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08785123378038406, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.05048300698399544, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.007729661650955677, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10795362293720245, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006207324913702905, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.07816172391176224, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.004719503223896027, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7729273438453674, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59607.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3347315490245819, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.395867109298706, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.2651832699775696, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.16910108923912048, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10557225346565247, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 2.8542752265930176, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0231173038482666, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0967686176300049, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.05828468129038811, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.615314483642578, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72197.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8256045579910278, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 16.194381713867188, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 20.1199951171875, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 15.517733573913574, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.8476231694221497, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -4.314092636108398, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 10.70268726348877, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5463039875030518, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 19.543176651000977, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0278365109115839, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1183546632528305, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10481784492731094, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.006686676759272814, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1668366938829422, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08497651666402817, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.7543575763702393, "metrics/particle/momentum/all/accepted/pz_bias": 0.002007940085604787, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.171735763549805, "metrics/particle/momentum/all/accepted/samples": 248704.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0650601387023926, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0931732654571533, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6961479783058167, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5245294570922852, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.24027781188488007, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.3956239223480225, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.23601064085960388, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.894759178161621, "metrics/particle/momentum/all/assigned/pz_bias": -0.023994099348783493, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.2578067779541, "metrics/particle/momentum/all/assigned/samples": 388313.0, "metrics/particle/momentum/all/efficiency": 0.635811448097229, "metrics/particle/momentum/all/event/energy_bias": 417.2353515625, "metrics/particle/momentum/all/event/p3_error_mean": 382.2149963378906, "metrics/particle/momentum/all/event/pt_vector_error_mean": 26.959779739379883, "metrics/particle/momentum/all/event/px_bias": 0.02209794521331787, "metrics/particle/momentum/all/event/py_bias": -9.177080154418945, "metrics/particle/momentum/all/event/pz_abs_mean": 379.0433044433594, "metrics/particle/momentum/all/event/pz_bias": 33.780235290527344, "metrics/particle/momentum/all/event/pz_residual_rms": 549.9964599609375, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01413217093795538, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06555681675672531, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05142967030405998, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0030993286054581404, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09568092226982117, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001225039828568697, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.20253221690654755, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.006353296805173159, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.0035898685455322, "metrics/particle/momentum/class_1/accepted/samples": 171423.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.22825832664966583, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.915303111076355, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3189736604690552, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2577463388442993, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07748926430940628, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.325528621673584, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.023144593462347984, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.158902883529663, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.020395413041114807, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.648293495178223, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8891788125038147, "metrics/particle/momentum/class_1/event/energy_bias": 47.01245880126953, "metrics/particle/momentum/class_1/event/p3_error_mean": 55.13236618041992, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.972848892211914, "metrics/particle/momentum/class_1/event/px_bias": -1.337437391281128, "metrics/particle/momentum/class_1/event/py_bias": -1.0268545150756836, "metrics/particle/momentum/class_1/event/pz_abs_mean": 49.85655212402344, "metrics/particle/momentum/class_1/event/pz_bias": -9.178908348083496, "metrics/particle/momentum/class_1/event/pz_residual_rms": 79.6366958618164, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.058451756834983826, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23686917126178741, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23285052180290222, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07027605921030045, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27318075299263, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.10819222778081894, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.1648454666137695, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.1305958479642868, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 9.638076782226562, "metrics/particle/momentum/class_2/accepted/samples": 8032.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.176135301589966, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.4952597618103027, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1109862327575684, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6260877251625061, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.47184279561042786, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.9772274494171143, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.352641761302948, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.683786392211914, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.06396972388029099, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 35.0482063293457, "metrics/particle/momentum/class_2/assigned/samples": 31040.0, "metrics/particle/momentum/class_2/efficiency": 0.2585463225841522, "metrics/particle/momentum/class_2/event/energy_bias": -312.5003967285156, "metrics/particle/momentum/class_2/event/p3_error_mean": 172.62522888183594, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 28.53638458251953, "metrics/particle/momentum/class_2/event/px_bias": 0.9654412269592285, "metrics/particle/momentum/class_2/event/py_bias": -6.026729106903076, "metrics/particle/momentum/class_2/event/pz_abs_mean": 166.93038940429688, "metrics/particle/momentum/class_2/event/pz_bias": 3.1392457485198975, "metrics/particle/momentum/class_2/event/pz_residual_rms": 232.38829040527344, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.06074606999754906, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.22692519426345825, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.22390349209308624, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.004638641141355038, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.26325273513793945, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.31733790040016174, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.421076774597168, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.058479711413383484, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 14.023446083068848, "metrics/particle/momentum/class_3/accepted/samples": 21006.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.043996572494507, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.9676170945167542, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.1015657186508179, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.9277709126472473, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4232228398323059, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 2.888768196105957, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5386126041412354, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.470579147338867, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03912702575325966, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.621158599853516, "metrics/particle/momentum/class_3/assigned/samples": 52457.0, "metrics/particle/momentum/class_3/efficiency": 0.39410141110420227, "metrics/particle/momentum/class_3/event/energy_bias": 493.19976806640625, "metrics/particle/momentum/class_3/event/p3_error_mean": 281.7563171386719, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.598377227783203, "metrics/particle/momentum/class_3/event/px_bias": -0.5616863965988159, "metrics/particle/momentum/class_3/event/py_bias": -0.5913455486297607, "metrics/particle/momentum/class_3/event/pz_abs_mean": 281.3399353027344, "metrics/particle/momentum/class_3/event/pz_bias": -13.061616897583008, "metrics/particle/momentum/class_3/event/pz_residual_rms": 447.0431213378906, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.059679511934518814, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.2290622591972351, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22611746191978455, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.052597757428884506, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.26531335711479187, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.3581368625164032, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 11.035392761230469, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.05666825920343399, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.521629333496094, "metrics/particle/momentum/class_4/accepted/samples": 36648.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.8604627847671509, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0850012302398682, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.067825436592102, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8026721477508545, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.36851438879966736, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.151279926300049, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5311418175697327, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.34496307373047, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.08243660628795624, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 39.73786163330078, "metrics/particle/momentum/class_4/assigned/samples": 82317.0, "metrics/particle/momentum/class_4/efficiency": 0.4348156154155731, "metrics/particle/momentum/class_4/event/energy_bias": 186.87060546875, "metrics/particle/momentum/class_4/event/p3_error_mean": 333.0432434082031, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 10.758113861083984, "metrics/particle/momentum/class_4/event/px_bias": 1.3889613151550293, "metrics/particle/momentum/class_4/event/py_bias": -0.10835064947605133, "metrics/particle/momentum/class_4/event/pz_abs_mean": 332.5577392578125, "metrics/particle/momentum/class_4/event/pz_bias": 66.73542785644531, "metrics/particle/momentum/class_4/event/pz_residual_rms": 488.1131286621094, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.050965648144483566, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.28198111057281494, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21426621079444885, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03829577565193176, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.254662424325943, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.023289931938052177, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.281503677368164, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.020582469180226326, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.2311296463012695, "metrics/particle/momentum/class_5/accepted/samples": 10906.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.432956337928772, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.1088016033172607, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9808692932128906, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.678460955619812, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.3843522071838379, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.540013790130615, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.14645609259605408, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.4471516609191895, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.09308800846338272, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 17.637495040893555, "metrics/particle/momentum/class_5/assigned/samples": 28937.0, "metrics/particle/momentum/class_5/efficiency": 0.37675753235816956, "metrics/particle/momentum/class_5/event/energy_bias": 13.058643341064453, "metrics/particle/momentum/class_5/event/p3_error_mean": 112.73065185546875, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 25.335451126098633, "metrics/particle/momentum/class_5/event/px_bias": 0.13414499163627625, "metrics/particle/momentum/class_5/event/py_bias": -0.6863000988960266, "metrics/particle/momentum/class_5/event/pz_abs_mean": 105.098876953125, "metrics/particle/momentum/class_5/event/pz_bias": -12.50151538848877, "metrics/particle/momentum/class_5/event/pz_residual_rms": 164.72142028808594, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.019031085073947906, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.09554409980773926, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.09517981112003326, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.04884515330195427, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.13968336582183838, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.016129031777381897, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.8586779832839966, "metrics/particle/momentum/class_6/accepted/pz_bias": -0.06681440770626068, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 3.749058961868286, "metrics/particle/momentum/class_6/accepted/samples": 496.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.31257954239845276, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 0.8920138478279114, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.45921894907951355, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.31216081976890564, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.09913043677806854, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 2.1371591091156006, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.038260869681835175, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 1.8939921855926514, "metrics/particle/momentum/class_6/assigned/pz_bias": 0.17643511295318604, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 7.744783878326416, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8626086711883545, "metrics/particle/momentum/class_6/event/energy_bias": -8.81009292602539, "metrics/particle/momentum/class_6/event/p3_error_mean": 8.23621940612793, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 3.10740327835083, "metrics/particle/momentum/class_6/event/px_bias": -0.45589500665664673, "metrics/particle/momentum/class_6/event/py_bias": -0.618006706237793, "metrics/particle/momentum/class_6/event/pz_abs_mean": 7.167080879211426, "metrics/particle/momentum/class_6/event/pz_bias": -1.0905965566635132, "metrics/particle/momentum/class_6/event/pz_residual_rms": 20.614713668823242, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.01321822963654995, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.05521712079644203, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.04180330038070679, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.009187137708067894, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.08422167599201202, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.010362694039940834, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.6365730166435242, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.265301376581192, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 2.696657419204712, "metrics/particle/momentum/class_7/accepted/samples": 193.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.0460592545568943, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.06755637377500534, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.05786473676562309, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.018554622307419777, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.015075377188622952, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.15257200598716736, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.01005025114864111, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.6580901741981506, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.22696658968925476, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 2.680356740951538, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9698492288589478, "metrics/particle/momentum/class_7/event/energy_bias": -1.595679759979248, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.2550172805786133, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 0.99483323097229, "metrics/particle/momentum/class_7/event/px_bias": -0.11143075674772263, "metrics/particle/momentum/class_7/event/py_bias": -0.11949430406093597, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.8559088706970215, "metrics/particle/momentum/class_7/event/pz_bias": -0.26180416345596313, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.999582767486572, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 3.7335333824157715, "step_valid_time": 16697.815959692, "step_total_time": 16701.549493551254, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0505610722478955, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.362649880714933, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84463, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16878, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13670, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.20715556919300399, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.1998271432461551, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.2034253757427473, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8001728567538449, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.16778152807609698, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1618460154150338, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.1647603321722571, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.20715556919300399, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0524711078218156, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3819678815292573, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84381, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15803, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12618, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19452479720331367, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18728149701947122, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19083444028498975, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8127185029805288, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15531948940779675, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14953603299320936, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15237290182345128, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19452479720331367, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0544633160445562, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3915595560782025, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 83721, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15179, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12088, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18788448922501827, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.1813045711350796, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18453589447450003, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8186954288649204, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14962433004493186, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14438432412417435, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14695763175490853, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18788448922501827} \ No newline at end of file diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_90000.json b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_90000.json new file mode 100644 index 0000000000000000000000000000000000000000..4dee66f716a8e12974e12be1a84f00486505acd5 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/history/step_90000.json @@ -0,0 +1 @@ +{"epoch": 0.3878340759637677, "train": {"Classification_binary": 2.928194046020508, "Classification": 0.014930029399693012, "Regression_pt": 0.21777962148189545, "Regression_eta": 0.10438096523284912, "Regression_sin_phi": 0.04713018238544464, "Regression_cos_phi": 0.04761872813105583, "Regression_energy": 0.2655681371688843, "Total": 10.588778495788574, "Total_unweighted": 3.62560170982033}, "train_group": {"physical_pu": {"Classification_binary": 2.9182825009901445, "Classification": 0.015553313266526867, "Regression_pt": 0.22919334258352006, "Regression_eta": 0.10488055793793646, "Regression_sin_phi": 0.047142311275660334, "Regression_cos_phi": 0.04722207825589966, "Regression_energy": 0.27312481829098295, "Total": 10.702289696578141, "Total_unweighted": 3.635398922600671}, "physical_nopu": {"Classification_binary": 3.4713698243937476, "Classification": 0.014400541001666498, "Regression_pt": 0.3700858432106977, "Regression_eta": 0.0689098551148104, "Regression_sin_phi": 0.04737730761959214, "Regression_cos_phi": 0.04771740939488636, "Regression_energy": 0.4154996875554684, "Total": 12.550249312708218, "Total_unweighted": 4.435360468290869}}, "valid": {"Classification_binary": 3.1930853525797525, "Classification": 0.014983318746089935, "Regression_pt": 0.30139561494191486, "Regression_eta": 0.08780114849408467, "Regression_sin_phi": 0.04756168027718862, "Regression_cos_phi": 0.047526086370150246, "Regression_energy": 0.3459781805674235, "Total": 11.667648315429688, "group/physical_pu/Classification": 0.015523030112187067, "group/physical_pu/Classification_binary": 2.9156618118286133, "group/physical_pu/Regression_cos_phi": 0.04733816782633463, "group/physical_pu/Regression_energy": 0.2733185291290283, "group/physical_pu/Regression_eta": 0.10479511817296346, "group/physical_pu/Regression_pt": 0.22920175393422446, "group/physical_pu/Regression_sin_phi": 0.047298560539881386, "group/physical_pu/Total": 10.708580017089844, "group/physical_pu/Total_unweighted": 3.6331369715432325, "group/physical_nopu/Classification": 0.01444360613822937, "group/physical_nopu/Classification_binary": 3.4705092112223306, "group/physical_nopu/Regression_cos_phi": 0.04771400491396586, "group/physical_nopu/Regression_energy": 0.41863783200581867, "group/physical_nopu/Regression_eta": 0.07080718378225963, "group/physical_nopu/Regression_pt": 0.3735894362131755, "group/physical_nopu/Regression_sin_phi": 0.04782480001449585, "group/physical_nopu/Total": 12.626714070638021, "group/physical_nopu/Total_unweighted": 4.443526074290276, "Total_unweighted": 4.0383313819766045, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.014755993150174618, "diagnostic/loss/cms_pf/Classification_binary": 3.186612129211426, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.046990226954221725, "diagnostic/loss/cms_pf/Regression_energy": 0.3341338038444519, "diagnostic/loss/cms_pf/Regression_eta": 0.08796656876802444, "diagnostic/loss/cms_pf/Regression_pt": 0.2894393801689148, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.045668575912714005, "diagnostic/regression/cms_pf/energy_pred_mean": 0.21952641010284424, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.3556242883205414, "diagnostic/regression/cms_pf/energy_residual_mean": 0.10876018553972244, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5840620398521423, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.11074348539113998, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.3265446722507477, "diagnostic/regression/cms_pf/pt_residual_mean": 0.1215156689286232, "diagnostic/regression/cms_pf/pt_residual_rms": 0.543615460395813, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.890522837638855, "metrics/particle/class_1/pid_accuracy": 0.9785614609718323, "metrics/particle/class_2/efficiency": 0.2562141418457031, "metrics/particle/class_2/pid_accuracy": 0.20837558805942535, "metrics/particle/class_3/efficiency": 0.39554885029792786, "metrics/particle/class_3/pid_accuracy": 0.47516074776649475, "metrics/particle/class_4/efficiency": 0.4384819269180298, "metrics/particle/class_4/pid_accuracy": 0.6602915525436401, "metrics/particle/class_5/efficiency": 0.372551828622818, "metrics/particle/class_5/pid_accuracy": 0.6330592632293701, "metrics/particle/class_6/efficiency": 0.8635397553443909, "metrics/particle/class_6/pid_accuracy": 0.05349523946642876, "metrics/particle/class_7/efficiency": 0.9638602137565613, "metrics/particle/class_7/pid_accuracy": 0.2521631717681885, "metrics/particle/count/bias_mean": 64.10221099853516, "metrics/particle/count/mae": 86.291015625, "metrics/particle/count/prediction_mean": 1592.7056884765625, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.08594626933336258, "metrics/particle/event/energy_response_mean": 0.9412158131599426, "metrics/particle/event/met_abs_error": 18.326875686645508, "metrics/particle/event/scalar_pt_relative_abs_error": 0.046071797609329224, "metrics/particle/event/scalar_pt_response_mean": 1.0022977590560913, "metrics/particle/event/vector_pt_closure": 0.02740365080535412, "metrics/particle/matched/delta_eta_abs_mean": 0.019113751128315926, "metrics/particle/matched/delta_phi_abs_mean": 0.015982020646333694, "metrics/particle/matched/delta_r_mean": 0.02793942764401436, "metrics/particle/matched/energy_relative_abs_mean": 0.1178775355219841, "metrics/particle/matched/pid_accuracy": 0.8464253544807434, "metrics/particle/matched/pt_relative_abs_mean": 0.10490008443593979, "metrics/particle/matching/duplicate_fraction": 0.11046106368303299, "metrics/particle/matching/efficiency": 0.6367058753967285, "metrics/particle/matching/f1": 0.6236298680305481, "metrics/particle/matching/purity": 0.6110801100730896, "metrics/particle/matching/target_coverage_dr0p05": 0.5126854181289673, "metrics/particle/matching/target_coverage_dr0p10": 0.7006571888923645, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.557969868183136, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7554402351379395, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017607878893613815, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06903287768363953, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.057533759623765945, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.002386086154729128, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.10917994379997253, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0019168300786986947, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.2194845974445343, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 9.567866800352931e-05, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.3556263446807861, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68342.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.47773781418800354, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1566216945648193, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.37527233362197876, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.2475401908159256, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1379963457584381, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.3111424446105957, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.04039251431822777, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8707484006881714, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.10310231149196625, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.52160930633545, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87640.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7797058820724487, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 50.65865707397461, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 43.63193893432617, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 16.031959533691406, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.3957422971725464, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.06599052250385284, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 38.35747528076172, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -3.3385581970214844, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 59.362762451171875, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.040722329169511795, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16097694635391235, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15818020701408386, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.018571779131889343, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21161819994449615, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17341923713684082, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.484125137329102, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.035050224512815475, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.608508110046387, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 121065.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5150591135025024, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9654032588005066, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9326610565185547, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7160751223564148, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.31946030259132385, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3.429213523864746, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.3771267831325531, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.736394882202148, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.038820862770080566, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.21931838989258, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228573.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5233862400054932, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 240.146484375, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 383.0477294921875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 19.331100463867188, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -1.4159424304962158, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.5987796783447266, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 381.1558532714844, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 63.110233306884766, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 556.37939453125, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013320057652890682, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08527212589979172, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0495896115899086, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008535333909094334, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10620192438364029, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006212953012436628, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.078394815325737, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005609821528196335, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7979085445404053, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59553.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3448331952095032, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.396039605140686, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.26140642166137695, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.1637197881937027, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10494896024465561, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 2.298398017883301, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.023740598931908607, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.1241734027862549, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.06161300092935562, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.8222527503967285, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72197.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8248566389083862, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 10.099143028259277, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 20.574678421020508, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 15.614068031311035, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 4.146737575531006, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.0745813846588135, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 11.22700309753418, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.8492670655250549, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 20.79637336730957, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02782236412167549, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11762824654579163, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10457610338926315, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.007644431199878454, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16657449305057526, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08500562608242035, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.745839834213257, "metrics/particle/momentum/all/accepted/pz_bias": 0.01572868414223194, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.135540962219238, "metrics/particle/momentum/all/accepted/samples": 248960.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0634804964065552, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0885953903198242, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6821213960647583, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5076850056648254, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.238642156124115, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.017850875854492, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.23545995354652405, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.891685485839844, "metrics/particle/momentum/all/assigned/pz_bias": -0.011870845220983028, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.298297882080078, "metrics/particle/momentum/all/assigned/samples": 388410.0, "metrics/particle/momentum/all/efficiency": 0.6364659070968628, "metrics/particle/momentum/all/event/energy_bias": 300.904296875, "metrics/particle/momentum/all/event/p3_error_mean": 381.5947570800781, "metrics/particle/momentum/all/event/pt_vector_error_mean": 25.937686920166016, "metrics/particle/momentum/all/event/px_bias": 4.126537799835205, "metrics/particle/momentum/all/event/py_bias": -0.4098111391067505, "metrics/particle/momentum/all/event/pz_abs_mean": 377.93450927734375, "metrics/particle/momentum/all/event/pz_bias": 58.92240524291992, "metrics/particle/momentum/all/event/pz_residual_rms": 552.0425415039062, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.014113281853497028, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06486496329307556, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05124467983841896, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00295901857316494, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0952703058719635, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011937273666262627, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.20292742550373077, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0035177015233784914, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.097326636314392, "metrics/particle/momentum/class_1/accepted/samples": 171731.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.22489680349826813, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.9045650959014893, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.31176507472991943, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.25037840008735657, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0759539008140564, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.708240032196045, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.02303047850728035, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1522704362869263, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02043488435447216, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.717743873596191, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8907763957977295, "metrics/particle/momentum/class_1/event/energy_bias": 49.26254653930664, "metrics/particle/momentum/class_1/event/p3_error_mean": 54.904354095458984, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 16.603328704833984, "metrics/particle/momentum/class_1/event/px_bias": 0.3543739318847656, "metrics/particle/momentum/class_1/event/py_bias": -1.8838610649108887, "metrics/particle/momentum/class_1/event/pz_abs_mean": 48.818023681640625, "metrics/particle/momentum/class_1/event/pz_bias": -6.624919414520264, "metrics/particle/momentum/class_1/event/pz_residual_rms": 78.8352279663086, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.05845172703266144, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23742195963859558, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23320412635803223, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0713503360748291, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27279186248779297, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.10769423842430115, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.221679210662842, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.11824546754360199, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 9.948854446411133, "metrics/particle/momentum/class_2/accepted/samples": 7967.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.2054967880249023, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.505403757095337, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.072524070739746, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5821980834007263, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4713839292526245, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.2324788570404053, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.3523784875869751, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.733579635620117, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.14438307285308838, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 35.112403869628906, "metrics/particle/momentum/class_2/assigned/samples": 31049.0, "metrics/particle/momentum/class_2/efficiency": 0.2564539909362793, "metrics/particle/momentum/class_2/event/energy_bias": -298.36358642578125, "metrics/particle/momentum/class_2/event/p3_error_mean": 171.2263641357422, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 28.50882339477539, "metrics/particle/momentum/class_2/event/px_bias": 2.220332145690918, "metrics/particle/momentum/class_2/event/py_bias": -2.393150806427002, "metrics/particle/momentum/class_2/event/pz_abs_mean": 165.09185791015625, "metrics/particle/momentum/class_2/event/pz_bias": -30.532411575317383, "metrics/particle/momentum/class_2/event/pz_residual_rms": 232.67405700683594, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.06073139235377312, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.2265271544456482, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.22343851625919342, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.006827888078987598, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.262935072183609, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.3149043023586273, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.361148834228516, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0015823377761989832, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 13.944746017456055, "metrics/particle/momentum/class_3/accepted/samples": 21108.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.023070812225342, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.9669579267501831, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.0737038850784302, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.8954811096191406, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.42052993178367615, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 2.767723560333252, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5361990332603455, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.335689544677734, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.1114037036895752, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.29478454589844, "metrics/particle/momentum/class_3/assigned/samples": 52460.0, "metrics/particle/momentum/class_3/efficiency": 0.39601507782936096, "metrics/particle/momentum/class_3/event/energy_bias": 594.9113159179688, "metrics/particle/momentum/class_3/event/p3_error_mean": 287.33746337890625, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.532122611999512, "metrics/particle/momentum/class_3/event/px_bias": 0.19106623530387878, "metrics/particle/momentum/class_3/event/py_bias": -0.27913063764572144, "metrics/particle/momentum/class_3/event/pz_abs_mean": 286.9891052246094, "metrics/particle/momentum/class_3/event/pz_bias": -2.998892307281494, "metrics/particle/momentum/class_3/event/pz_residual_rms": 452.7314453125, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.05972544848918915, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.22975532710552216, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22683891654014587, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.056511685252189636, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2661703824996948, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.3601670563220978, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 11.005668640136719, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1027202159166336, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.392473220825195, "metrics/particle/momentum/class_4/accepted/samples": 36633.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.8543840646743774, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0792278051376343, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0561968088150024, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.7850003838539124, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.36685556173324585, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.2203609943389893, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5294710397720337, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.36896324157715, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.04031621292233467, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 39.93458557128906, "metrics/particle/momentum/class_4/assigned/samples": 82403.0, "metrics/particle/momentum/class_4/efficiency": 0.4346376657485962, "metrics/particle/momentum/class_4/event/energy_bias": -44.345970153808594, "metrics/particle/momentum/class_4/event/p3_error_mean": 337.6934509277344, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 10.88068962097168, "metrics/particle/momentum/class_4/event/px_bias": -0.5714081525802612, "metrics/particle/momentum/class_4/event/py_bias": 0.41145607829093933, "metrics/particle/momentum/class_4/event/pz_abs_mean": 337.03509521484375, "metrics/particle/momentum/class_4/event/pz_bias": 102.49835205078125, "metrics/particle/momentum/class_4/event/pz_residual_rms": 498.47735595703125, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.051229026168584824, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.27662593126296997, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21186380088329315, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.036119621247053146, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.252509742975235, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.022811230272054672, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2712585926055908, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.008454016409814358, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.13072395324707, "metrics/particle/momentum/class_5/accepted/samples": 10828.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4545753002166748, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.1104815006256104, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9604216814041138, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6527279615402222, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.3821882903575897, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.311645984649658, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.14798174798488617, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.5159196853637695, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.24531827867031097, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 17.805768966674805, "metrics/particle/momentum/class_5/assigned/samples": 28936.0, "metrics/particle/momentum/class_5/efficiency": 0.3740629553794861, "metrics/particle/momentum/class_5/event/energy_bias": 10.242776870727539, "metrics/particle/momentum/class_5/event/p3_error_mean": 112.20394134521484, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 25.434560775756836, "metrics/particle/momentum/class_5/event/px_bias": 2.6385586261749268, "metrics/particle/momentum/class_5/event/py_bias": 4.4981689453125, "metrics/particle/momentum/class_5/event/pz_abs_mean": 104.44352722167969, "metrics/particle/momentum/class_5/event/pz_bias": -2.643838882446289, "metrics/particle/momentum/class_5/event/pz_residual_rms": 163.75088500976562, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.019757041707634926, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.09817833453416824, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.09743108600378036, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.04458467662334442, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.14229387044906616, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.020040079951286316, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.9367578029632568, "metrics/particle/momentum/class_6/accepted/pz_bias": -0.035652052611112595, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 4.533817768096924, "metrics/particle/momentum/class_6/accepted/samples": 499.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.3351922631263733, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 1.5928173065185547, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.6405157446861267, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.49100568890571594, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.10260869562625885, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 2.8102426528930664, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.052173912525177, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 2.6991820335388184, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.9726707935333252, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 11.599939346313477, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8678261041641235, "metrics/particle/momentum/class_6/event/energy_bias": -9.313739776611328, "metrics/particle/momentum/class_6/event/p3_error_mean": 7.7916107177734375, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 3.090386152267456, "metrics/particle/momentum/class_6/event/px_bias": -0.63313227891922, "metrics/particle/momentum/class_6/event/py_bias": -0.6836140751838684, "metrics/particle/momentum/class_6/event/pz_abs_mean": 6.701862335205078, "metrics/particle/momentum/class_6/event/pz_bias": -0.5330881476402283, "metrics/particle/momentum/class_6/event/pz_residual_rms": 18.516048431396484, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.014821801334619522, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.05877865478396416, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.04237481206655502, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.013378617353737354, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.07873613387346268, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.010309278033673763, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.6332058310508728, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.21840304136276245, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 2.5651767253875732, "metrics/particle/momentum/class_7/accepted/samples": 194.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.05603589117527008, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.23992176353931427, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.09080632776021957, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.05193621292710304, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.015075377188622952, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.5606458187103271, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.015075377188622952, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.74305260181427, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.09221898019313812, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 2.8362722396850586, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9748743772506714, "metrics/particle/momentum/class_7/event/energy_bias": -1.4890261888504028, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.2101645469665527, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 0.9828690886497498, "metrics/particle/momentum/class_7/event/px_bias": -0.07325287163257599, "metrics/particle/momentum/class_7/event/py_bias": -0.07967982441186905, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.8047243356704712, "metrics/particle/momentum/class_7/event/pz_bias": -0.24279853701591492, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.894932746887207, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 4.81864070892334, "step_valid_time": 18781.826822519302, "step_total_time": 18786.64546394348, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0412791464853473, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3504564831057633, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84427, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 17059, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13939, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.20937710954280453, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.20205621424425835, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.2056515292160432, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.7979437857557417, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.17108315434182264, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.16510121169767966, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.16803896276114816, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.20937710954280453, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0408928416951155, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3762233459405624, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84278, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16007, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12875, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19703590639963564, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.1899309428320558, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19341819873487315, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8100690571679442, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1584829946208102, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15276821946415434, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15557314354416768, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19703590639963564, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0487666664810291, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.37904436130201125, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 83628, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15293, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12255, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18929557241703698, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18286937389391114, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18602699234264097, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8171306261060889, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1516914431420119, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14654182809585306, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14907217623481755, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18929557241703698} \ No newline at end of file diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/hyperparameters.json b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..30bd7a07a329326a1e228bfb04cafde57767a0da --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 12421153, "dataset": "cms", "data_dir": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": false, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 3, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "gnnlsh": null, "attention": {"num_convs": 3, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 32, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": true, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0004}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 55, "num_classes": 9, "elemtypes_nonzero": [1, 4, 5, 6, 8, 9, 10, 11], "seed": 12345, "num_steps": 100000, "patience": 10000, "checkpoint_freq": 10000, "val_freq": 10000, "num_workers": 2, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 8, "dtype": "bfloat16", "lr": 0.0004, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "cosinedecay", "lr_schedule_config": {}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 100, "validation_diagnostics_batches": 4, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cms": {"physical_pu": {"batch_size": 1, "weight": null, "samples": {"cms_pf_ttbar": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_qcd": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_ztt": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}, "physical_nopu": {"batch_size": 1, "weight": null, "samples": {"cms_pf_ttbar_nopu": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_qcd_nopu": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_ztt_nopu": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cms": {"physical_pu": {"batch_size": 1, "weight": null, "samples": {"cms_pf_ttbar": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_qcd": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_ztt": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}, "physical_nopu": {"batch_size": 1, "weight": null, "samples": {"cms_pf_ttbar_nopu": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_qcd_nopu": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_ztt_nopu": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cms_pf_ttbar": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "cms_pf_qcd": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "cms_pf_ztt": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cms_pf_ttbar", "cms_pf_qcd", "cms_pf_ztt"], "ntrain": null, "nvalid": 512, "ntest": 512, "data_config": null, "gpus": 8} \ No newline at end of file diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/model_kwargs.pkl b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..6c2160aaf9b35f0a5d8021adea4d06dbd2172420 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fefa3650c47ee05bd89597e8c23f918345d9f9a7a78fd6b9ee57b88a789e1f0 +size 5898 diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/particleflow_spec.yaml b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..764890baecded6715d37cf35f42c4079f6e00e2c --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/particleflow_spec.yaml @@ -0,0 +1,2551 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.0 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.0 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id019 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id020 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id019 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id020 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_eta.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..086b7d6a285abe9e56ee7aca3c717d872c12f6f6 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_eta.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_eta.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..06ed41c80748abd76d2e043cf0403ebde280245b Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_eta.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_pt.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e2a3e9328b573f1d1350e569d82dc080fff51f79 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_pt.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_pt.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..b59329c26da4f8459a43aec28758e0e80e00f777 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_pt.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_pt_log.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..37690a448e3f8ac527c5e8c88e8b166ab883508b Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_pt_log.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_pt_log.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..6dfd96930773eb98d6397dbbdbe3089f6e2b867e Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_pt_log.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_res.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9526a5ad025eeb9a09d7dc959f974daa79a5e665 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_res.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_res.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..f9c8a8b885264a4b270a78933c60da4b964b8543 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_res.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_res_vstarget.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5c1bbda0fed20fad7584d3a9de41f283b9f709de Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_res_vstarget.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_res_vstarget.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..50b53340233de00007031b703bb1e79578f62fb5 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/jet_res_vstarget.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/num_elements.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..28d1f28d7886f76926996491ceebc9bbcaebd454 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/num_elements.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/num_elements.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..140b43e3aea253ca07ed2318f9486f8a9f59382f Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_qcd/num_elements.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_eta.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..83c949b43a66f192f09d6f22ce025c705f0ba3b2 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_eta.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_eta.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..823c666e26ac61f0252f91d50429785de86f540e Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_eta.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_pt.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f85c384d10b979362d6b0e5571cabe902318577a Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_pt.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_pt.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..2010751c58b7f83019ff06cb533814355a13f46b Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_pt.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_pt_log.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9cdb0bfb1ea317c7fdb6adc5c00c34f3c92bd273 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_pt_log.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_pt_log.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..d1272fde0ed886a6dbfd61b2d69c14208c406654 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_pt_log.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_res.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3a56391c7ba8d5581a8f558be5f717a4d71a5219 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_res.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_res.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..b3730c7831e20f7e34b0e13473aab46d8c976c0e Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_res.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_res_vstarget.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fd1c1df3a4858998e60b8ba029389f78288ed306 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_res_vstarget.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_res_vstarget.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..0067f0df9ce8c4518fa5124585ba663da6defe6d Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/jet_res_vstarget.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/num_elements.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8fefde3620bc11ee16949bd1bb947511fddce08f Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/num_elements.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/num_elements.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..6ceb538c6430afebc3faddf55571a5d66c75ae1f Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ttbar/num_elements.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_eta.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d394828727a58bc42bd222b39cb3989d892b9402 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_eta.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_eta.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..7c31487dcbc4f2972264927e953e0578a0e47159 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_eta.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_pt.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b15dec450201a38f2e63484cd75cf2e7d5858a56 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_pt.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_pt.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..80283be9609fa2669f408a25677280335817cf03 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_pt.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_pt_log.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5596b343fefd18fb0c9917baaa215301042c3172 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_pt_log.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_pt_log.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..e6d97244cd5e96424cbe8cfd63aa1efc84e2568e Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_pt_log.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_res.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fbfad8c0e6fa00af3664f7cfbe745734c6aac62a Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_res.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_res.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..08d446cd3a38517efa9416e5255cf44c1559c2c5 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_res.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_res_vstarget.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6856b35432ab3962383ca2fe2896f6e1a8644f05 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_res_vstarget.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_res_vstarget.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..9e4002c52378bfbb2519eb7a926d9b4c0fb95fe6 Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/jet_res_vstarget.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/num_elements.pdf b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9818d626310aa222993459b8a6e52491a8084f5b Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/num_elements.pdf differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/num_elements.png b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..7f04e088562285379e5949e64677b79cdc285d1f Binary files /dev/null and b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/plots_step_100000/cms_pf_ztt/num_elements.png differ diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/runs/train/events.out.tfevents.1793110276.nid007557.49823.0 b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/runs/train/events.out.tfevents.1793110276.nid007557.49823.0 new file mode 100644 index 0000000000000000000000000000000000000000..aa5385299f50fde865f6f5b8462c3f630537c92d --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/runs/train/events.out.tfevents.1793110276.nid007557.49823.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f8844573315bad5408436967b0a23372ef05cbcbb52282ee6e2d888264a7dc5 +size 8928694 diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/runs/valid/events.out.tfevents.1793110276.nid007557.49823.1 b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/runs/valid/events.out.tfevents.1793110276.nid007557.49823.1 new file mode 100644 index 0000000000000000000000000000000000000000..ec6658f68c1e7c8216eb224a5681e69fa01e6403 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/runs/valid/events.out.tfevents.1793110276.nid007557.49823.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91e279b4396103202b10d5cb42f8afe06bf425620ac6453c3129eb39921714e2 +size 15785401 diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/scenario-manifest.json b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..c0bcf0cdeec0fc0ce9932d54b2e1770267174d06 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/scenario-manifest.json @@ -0,0 +1,715 @@ +{ + "scenario": { + "name": "cms_backbone_comparison", + "spec_file": "particleflow_spec.yaml", + "production_name": "cms_run3", + "variants": { + "attention": { + "model_name": "pyg-cms-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.attention.num_heads": 16, + "model.attention.head_dim": 32 + } + }, + "heptv2": { + "model_name": "pyg-cms-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 512, + "model.heptv2.width": 512, + "model.heptv2.num_convs": 3, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 100, + "model.heptv2.n_hashes": 3, + "model.heptv2.deterministic_hashing": true + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 64, + "parameters": { + "lr": 0.0004, + "num_steps": 100000, + "val_freq": 10000, + "checkpoint_freq": 10000, + "tensorboard_step_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "pad_to_multiple_elements": 100, + "make_plots": true + } + }, + "common_overrides": { + "model.output_mode": "elementwise", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 3, + "train_dataset.cms.physical_nopu.batch_size": 1, + "valid_dataset.cms.physical_nopu.batch_size": 1, + "train_dataset.cms.physical_pu.samples.cms_pf_ttbar.version": "2.8.0", + "train_dataset.cms.physical_pu.samples.cms_pf_qcd.version": "2.8.0", + "train_dataset.cms.physical_pu.samples.cms_pf_ztt.version": "2.8.0", + "train_dataset.cms.physical_nopu.samples.cms_pf_ttbar_nopu.version": "2.8.0", + "train_dataset.cms.physical_nopu.samples.cms_pf_qcd_nopu.version": "2.8.0", + "train_dataset.cms.physical_nopu.samples.cms_pf_ztt_nopu.version": "2.8.0", + "valid_dataset.cms.physical_pu.samples.cms_pf_ttbar.version": "2.8.0", + "valid_dataset.cms.physical_pu.samples.cms_pf_qcd.version": "2.8.0", + "valid_dataset.cms.physical_pu.samples.cms_pf_ztt.version": "2.8.0", + "valid_dataset.cms.physical_nopu.samples.cms_pf_ttbar_nopu.version": "2.8.0", + "valid_dataset.cms.physical_nopu.samples.cms_pf_qcd_nopu.version": "2.8.0", + "valid_dataset.cms.physical_nopu.samples.cms_pf_ztt_nopu.version": "2.8.0", + "test_dataset.cms_pf_ttbar.version": "2.8.0", + "test_dataset.cms_pf_qcd.version": "2.8.0", + "test_dataset.cms_pf_ztt.version": "2.8.0" + }, + "allowed_variant_differences": [ + "conv_type", + "model.type", + "model.attention", + "model.heptv2" + ] + }, + "platform": { + "name": "lumi_mi250x", + "gpus": 8, + "python_command": [ + "python3" + ], + "data_dir": { + "cld": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "cms_run3": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/" + }, + "experiments_dir": "/scratch/project_465001293/patajoos/particleflow/experiments", + "environment": { + "MIOPEN_USER_DB_PATH": "/tmp/patajoos-23218977-miopen-cache", + "MIOPEN_CUSTOM_CACHE_DIR": "/tmp/patajoos-23218977-miopen-cache", + "ROCM_PATH": "/opt/rocm", + "KERAS_BACKEND": "torch", + "OMP_NUM_THREADS": "4", + "PYTHONUNBUFFERED": "1", + "NCCL_SOCKET_IFNAME": "hsn", + "NCCL_NET_GDR_LEVEL": "3" + }, + "runtime_overrides": { + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true + }, + "slurm": { + "partition": "standard-g", + "constraint": null, + "account": "project_465001293", + "time": "2-00:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 32, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "480G", + "memory_per_gpu": null, + "no_requeue": true + } + }, + "job": { + "scenario_name": "cms_backbone_comparison", + "platform_name": "lumi_mi250x", + "variant_name": "attention", + "model_name": "pyg-cms-v1", + "production_name": "cms_run3", + "data_dir": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/", + "seed": 12345, + "global_batch_size": 64, + "per_gpu_batch_size": 8, + "gpu_batch_multiplier": 8, + "settings": { + "lr": 0.0004, + "num_steps": 100000, + "val_freq": 10000, + "checkpoint_freq": 10000, + "tensorboard_step_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "pad_to_multiple_elements": 100, + "make_plots": true, + "model.output_mode": "elementwise", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 3, + "train_dataset.cms.physical_nopu.batch_size": 1, + "valid_dataset.cms.physical_nopu.batch_size": 1, + "train_dataset.cms.physical_pu.samples.cms_pf_ttbar.version": "2.8.0", + "train_dataset.cms.physical_pu.samples.cms_pf_qcd.version": "2.8.0", + "train_dataset.cms.physical_pu.samples.cms_pf_ztt.version": "2.8.0", + "train_dataset.cms.physical_nopu.samples.cms_pf_ttbar_nopu.version": "2.8.0", + "train_dataset.cms.physical_nopu.samples.cms_pf_qcd_nopu.version": "2.8.0", + "train_dataset.cms.physical_nopu.samples.cms_pf_ztt_nopu.version": "2.8.0", + "valid_dataset.cms.physical_pu.samples.cms_pf_ttbar.version": "2.8.0", + "valid_dataset.cms.physical_pu.samples.cms_pf_qcd.version": "2.8.0", + "valid_dataset.cms.physical_pu.samples.cms_pf_ztt.version": "2.8.0", + "valid_dataset.cms.physical_nopu.samples.cms_pf_ttbar_nopu.version": "2.8.0", + "valid_dataset.cms.physical_nopu.samples.cms_pf_qcd_nopu.version": "2.8.0", + "valid_dataset.cms.physical_nopu.samples.cms_pf_ztt_nopu.version": "2.8.0", + "test_dataset.cms_pf_ttbar.version": "2.8.0", + "test_dataset.cms_pf_qcd.version": "2.8.0", + "test_dataset.cms_pf_ztt.version": "2.8.0", + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true, + "model.attention.num_heads": 16, + "model.attention.head_dim": 32, + "seed": 12345, + "gpu_batch_multiplier": 8 + } + }, + "resolved_config": { + "dataset": "cms", + "data_dir": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": false, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 3, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "gnnlsh": null, + "attention": { + "num_convs": 3, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 32, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": true, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0004 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 55, + "num_classes": 9, + "elemtypes_nonzero": [ + 1, + 4, + 5, + 6, + 8, + 9, + 10, + 11 + ], + "seed": 12345, + "num_steps": 100000, + "patience": 10000, + "checkpoint_freq": 10000, + "val_freq": 10000, + "num_workers": 2, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 8, + "dtype": "bfloat16", + "lr": 0.0004, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "cosinedecay", + "lr_schedule_config": {}, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 100, + "validation_diagnostics_batches": 4, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cms": { + "physical_pu": { + "batch_size": 1, + "weight": null, + "samples": { + "cms_pf_ttbar": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_qcd": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_ztt": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + }, + "physical_nopu": { + "batch_size": 1, + "weight": null, + "samples": { + "cms_pf_ttbar_nopu": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_qcd_nopu": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_ztt_nopu": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cms": { + "physical_pu": { + "batch_size": 1, + "weight": null, + "samples": { + "cms_pf_ttbar": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_qcd": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_ztt": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + }, + "physical_nopu": { + "batch_size": 1, + "weight": null, + "samples": { + "cms_pf_ttbar_nopu": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_qcd_nopu": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_ztt_nopu": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cms_pf_ttbar": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "cms_pf_qcd": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "cms_pf_ztt": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cms_pf_ttbar", + "cms_pf_qcd", + "cms_pf_ztt" + ], + "ntrain": null, + "nvalid": 512, + "ntest": 512, + "data_config": null, + "gpus": 8 + }, + "command": [ + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cms-v1", + "--production-name", + "cms_run3", + "--data-dir", + "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/", + "--experiment-dir", + "/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225", + "train", + "--gpus", + "8", + "--dtype", + "bfloat16", + "--make-plots", + "--lr", + "0.0004", + "--num_steps", + "100000", + "--val_freq", + "10000", + "--checkpoint_freq", + "10000", + "--tensorboard_step_freq", + "1000", + "--nvalid", + "512", + "--ntest", + "512", + "--sampler_mode", + "interleaved-shards", + "--validation_diagnostics_batches", + "4", + "--pad_to_multiple_elements", + "100", + "--model.output_mode", + "elementwise", + "--model.task_queries", + "false", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "3", + "--train_dataset.cms.physical_nopu.batch_size", + "1", + "--valid_dataset.cms.physical_nopu.batch_size", + "1", + "--train_dataset.cms.physical_pu.samples.cms_pf_ttbar.version", + "2.8.0", + "--train_dataset.cms.physical_pu.samples.cms_pf_qcd.version", + "2.8.0", + "--train_dataset.cms.physical_pu.samples.cms_pf_ztt.version", + "2.8.0", + "--train_dataset.cms.physical_nopu.samples.cms_pf_ttbar_nopu.version", + "2.8.0", + "--train_dataset.cms.physical_nopu.samples.cms_pf_qcd_nopu.version", + "2.8.0", + "--train_dataset.cms.physical_nopu.samples.cms_pf_ztt_nopu.version", + "2.8.0", + "--valid_dataset.cms.physical_pu.samples.cms_pf_ttbar.version", + "2.8.0", + "--valid_dataset.cms.physical_pu.samples.cms_pf_qcd.version", + "2.8.0", + "--valid_dataset.cms.physical_pu.samples.cms_pf_ztt.version", + "2.8.0", + "--valid_dataset.cms.physical_nopu.samples.cms_pf_ttbar_nopu.version", + "2.8.0", + "--valid_dataset.cms.physical_nopu.samples.cms_pf_qcd_nopu.version", + "2.8.0", + "--valid_dataset.cms.physical_nopu.samples.cms_pf_ztt_nopu.version", + "2.8.0", + "--test_dataset.cms_pf_ttbar.version", + "2.8.0", + "--test_dataset.cms_pf_qcd.version", + "2.8.0", + "--test_dataset.cms_pf_ztt.version", + "2.8.0", + "--num_workers", + "2", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "true", + "--model.attention.num_heads", + "16", + "--model.attention.head_dim", + "32", + "--seed", + "12345", + "--gpu_batch_multiplier", + "8" + ], + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc" +} \ No newline at end of file diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train-config.yaml b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd84641ad730debf38f14aa91b5c6f687767e2f8 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train-config.yaml @@ -0,0 +1,376 @@ +backend: pytorch +batch_size: 1 +checkpoint_freq: 10000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /scratch/project_465001293/joosep/mlpf/cms/2.8.0/ +dataset: cms +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 4 +- 5 +- 6 +- 8 +- 9 +- 10 +- 11 +enabled_test_datasets: +- cms_pf_ttbar +- cms_pf_qcd +- cms_pf_ztt +gpu_batch_multiplier: 8 +gpu_type: l40 +gpus: 8 +hyperparameters: + batch_size: 1 + lr: 0.0004 +input_dim: 55 +load: null +load_weights_only: false +lr: 0.0004 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: cosinedecay +lr_schedule_config: {} +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 32 + layernorm: true + num_convs: 3 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: true + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 3 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: false + trainable: all + type: attention +ntest: 512 +ntrain: null +num_classes: 9 +num_steps: 100000 +num_workers: 2 +nvalid: 512 +optimizer: lamb +pad_to_multiple_elements: 100 +patience: 10000 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cms_pf_qcd: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ttbar: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ztt: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 +threads: 16 +train: true +train_dataset: + cms: + physical_nopu: + batch_size: 1 + samples: + cms_pf_qcd_nopu: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ttbar_nopu: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ztt_nopu: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + weight: null + physical_pu: + batch_size: 1 + samples: + cms_pf_qcd: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ttbar: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ztt: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + weight: null +val_freq: 10000 +valid_dataset: + cms: + physical_nopu: + batch_size: 1 + samples: + cms_pf_qcd_nopu: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ttbar_nopu: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ztt_nopu: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + weight: null + physical_pu: + batch_size: 1 + samples: + cms_pf_qcd: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ttbar: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ztt: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + weight: null +validation_diagnostics_batches: 4 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log new file mode 100644 index 0000000000000000000000000000000000000000..90fa09819f80202927b0f9cabf8116864ee79f36 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log @@ -0,0 +1,10 @@ +rank0 [2026-10-27 16:10:02,921] INFO : Will use torch.nn.parallel.DistributedDataParallel() and 8 gpus +rank0 [2026-10-27 16:10:03,121] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,121] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,121] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,121] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,121] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,121] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,121] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,121] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,121] INFO : Spawning DDP processes diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.0 b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..1ef2f9e3c6e8b97e1ea49f5d40c9c6d68b21f1aa --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.0 @@ -0,0 +1,3775 @@ +rank0 [2026-10-27 16:10:10,427] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-27 16:10:10,428] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-27 16:10:37,170] INFO : Instantiating model +rank0 [2026-10-27 16:10:37,170] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank0 [2026-10-27 16:10:37,170] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank0 [2026-10-27 16:10:37,201] INFO : Input encoding initialization took 0.03s +rank0 [2026-10-27 16:10:37,201] INFO : nn0 parameters: 2130944 +rank0 [2026-10-27 16:10:37,201] INFO : Initializing shared backbone layers, num_convs=3 +rank0 [2026-10-27 16:10:37,201] INFO : Initializing backbone layer 0 +rank0 [2026-10-27 16:10:37,201] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-27 16:10:37,212] INFO : Initializing backbone layer 1 +rank0 [2026-10-27 16:10:37,212] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-27 16:10:37,223] INFO : Initializing backbone layer 2 +rank0 [2026-10-27 16:10:37,223] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-27 16:10:37,233] INFO : Backbone initialization took 0.03s +rank0 [2026-10-27 16:10:37,234] INFO : backbone parameters: 4733952 +rank0 [2026-10-27 16:10:37,234] INFO : Initializing output DNNs +rank0 [2026-10-27 16:10:37,268] INFO : Output DNNs initialization took 0.03s +rank0 [2026-10-27 16:10:37,268] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-27 16:10:37,268] INFO : jagged_attention enabled=True +rank0 [2026-10-27 16:10:37,268] INFO : flash_attn_varlen enabled=True +rank0 [2026-10-27 16:10:37,268] INFO : task_queries enabled=False +rank0 [2026-10-27 16:10:37,268] INFO : classification_norm parameters: 1024 +rank0 [2026-10-27 16:10:37,268] INFO : regression_norm parameters: 1024 +rank0 [2026-10-27 16:10:37,268] INFO : classification_query parameters: 0 +rank0 [2026-10-27 16:10:37,268] INFO : regression_query parameters: 0 +rank0 [2026-10-27 16:10:37,268] INFO : classification_readout parameters: 0 +rank0 [2026-10-27 16:10:37,268] INFO : regression_readout parameters: 0 +rank0 [2026-10-27 16:10:37,268] INFO : output_mode=elementwise +rank0 [2026-10-27 16:10:37,269] INFO : set_decoder parameters: 0 +rank0 [2026-10-27 16:10:37,269] INFO : nn_binary_particle parameters: 264706 +rank0 [2026-10-27 16:10:37,269] INFO : nn_pid parameters: 268297 +rank0 [2026-10-27 16:10:37,269] INFO : nn_pt parameters: 2113544 +rank0 [2026-10-27 16:10:37,269] INFO : nn_eta parameters: 264706 +rank0 [2026-10-27 16:10:37,269] INFO : nn_sin_phi parameters: 264706 +rank0 [2026-10-27 16:10:37,269] INFO : nn_cos_phi parameters: 264706 +rank0 [2026-10-27 16:10:37,269] INFO : nn_energy parameters: 2113544 +rank0 [2026-10-27 16:10:37,270] INFO : Total MLPF parameters: 12421153 +rank0 [2026-10-27 16:10:37,270] INFO : MLPF __init__ done +rank0 [2026-10-27 16:10:37,270] INFO : Instantiated model +rank0 [2026-10-27 16:10:37,270] INFO : Moving model to device rank=0 +rank0 [2026-10-27 16:10:37,301] INFO : Moved model to device rank=0 +rank0 [2026-10-27 16:10:37,302] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-27 16:10:37,303] INFO : Configured model for SyncBatchNorm rank=0 +rank0 [2026-10-27 16:10:37,493] INFO : Configured model for DistributedDataParallel rank=0 +rank0 [2026-10-27 16:10:37,593] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +167 module.nn_energy.nn.7.0.bias ... 0 +168 module.nn_energy.nn.7.2.weight ... 0 +169 module.nn_energy.nn.7.2.bias ... 0 +170 module.nn_energy.nn.7.4.weight ... 0 +171 module.nn_energy.nn.7.4.bias ... 0 + +[172 rows x 3 columns] +rank0 [2026-10-27 16:10:37,593] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=512, out_features=512, bias=True) + ) + (norm0): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=512, out_features=512, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank0 [2026-10-27 16:10:37,594] INFO : Trainable parameters: 12421153 +rank0 [2026-10-27 16:10:37,594] INFO : Non-trainable parameters: 0 +rank0 [2026-10-27 16:10:37,594] INFO : Total parameters: 12421153 +rank0 [2026-10-27 16:10:37,596] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.mha.in_proj_weight 786432 0 + module.backbone.0.mha.in_proj_bias 1536 0 +module.backbone.0.mha.out_proj.weight 262144 0 + module.backbone.0.mha.out_proj.bias 512 0 + module.backbone.0.norm0.weight 512 0 + module.backbone.0.norm0.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm1.bias 512 0 + module.backbone.0.seq.0.weight 262144 0 + module.backbone.0.seq.0.bias 512 0 + module.backbone.0.seq.2.weight 262144 0 + module.backbone.0.seq.2.bias 512 0 + module.backbone.1.mha.in_proj_weight 786432 0 + module.backbone.1.mha.in_proj_bias 1536 0 +module.backbone.1.mha.out_proj.weight 262144 0 + module.backbone.1.mha.out_proj.bias 512 0 + module.backbone.1.norm0.weight 512 0 + module.backbone.1.norm0.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm1.bias 512 0 + module.backbone.1.seq.0.weight 262144 0 + module.backbone.1.seq.0.bias 512 0 + module.backbone.1.seq.2.weight 262144 0 + module.backbone.1.seq.2.bias 512 0 + module.backbone.2.mha.in_proj_weight 786432 0 + module.backbone.2.mha.in_proj_bias 1536 0 +module.backbone.2.mha.out_proj.weight 262144 0 + module.backbone.2.mha.out_proj.bias 512 0 + module.backbone.2.norm0.weight 512 0 + module.backbone.2.norm0.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm1.bias 512 0 + module.backbone.2.seq.0.weight 262144 0 + module.backbone.2.seq.0.bias 512 0 + module.backbone.2.seq.2.weight 262144 0 + module.backbone.2.seq.2.bias 512 0 + module.classification_norm.weight 512 0 + module.classification_norm.bias 512 0 + module.regression_norm.weight 512 0 + module.regression_norm.bias 512 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank0 [2026-10-27 16:10:37,601] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225 +rank0 [2026-10-27 16:10:37,601] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225 +rank0 [2026-10-27 16:10:37,601] INFO : Getting dataloaders +rank0 [2026-10-27 16:10:37,601] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:42,604] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:42,684] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:42,754] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:42,824] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:42,904] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:42,972] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:43,042] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:43,119] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:43,184] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:43,312] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:43,312] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:43,382] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:43,462] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:43,524] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:43,609] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:43,692] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:44,010] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:44,063] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:44,115] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:44,163] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:44,210] INFO : train_dataset: cms_pf_qcd, 45400 +rank0 [2026-10-27 16:10:44,210] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:44,257] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,303] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,350] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,410] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,480] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,547] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,614] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,683] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,746] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,827] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,827] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-27 16:10:44,828] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:44,930] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,030] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,142] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,238] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,335] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,437] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,806] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,887] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,968] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:46,052] INFO : train_dataset: cms_pf_ttbar_nopu, 450900 +rank0 [2026-10-27 16:10:46,052] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:46,133] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,209] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,313] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,387] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,509] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,599] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,687] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,787] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,884] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,992] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,992] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:47,065] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,133] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,482] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,538] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,591] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,644] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,695] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,742] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,791] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,839] INFO : train_dataset: cms_pf_ztt_nopu, 450900 +rank0 [2026-10-27 16:10:47,839] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-27 16:10:47,840] INFO : Creating WeightedMixIterator with 2 data loaders. +rank0 [2026-10-27 16:10:47,840] INFO : train mix: physical_pu: weight=21093 step_share=0.091 batches_per_epoch=21093 events_per_step=64 expected_passes_in_100000_steps=0.43 +rank0 [2026-10-27 16:10:47,840] INFO : train mix: physical_nopu: weight=210965 step_share=0.909 batches_per_epoch=210965 events_per_step=64 expected_passes_in_100000_steps=0.43 +rank0 [2026-10-27 16:10:47,840] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:47,855] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,869] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,884] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,899] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,916] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,931] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,947] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,962] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,977] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,992] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,992] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:48,006] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,021] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,036] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,051] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,067] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,082] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,096] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,110] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,124] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,139] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,139] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:48,154] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,171] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,189] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,204] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,220] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,235] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,250] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,265] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,280] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,295] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,295] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-27 16:10:48,295] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:48,312] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,328] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,344] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,361] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,378] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,394] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,411] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,427] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,444] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,463] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,463] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:48,484] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,500] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,515] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,532] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,549] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,565] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,582] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,599] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,616] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,632] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,632] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:48,647] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,663] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,679] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,695] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,710] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,726] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,741] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,756] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,771] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,785] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,786] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-27 16:10:48,786] INFO : Creating InterleavedIterator with 2 data loaders. +rank0 [2026-10-27 16:11:16,971] INFO : Got dataloaders +rank0 [2026-10-27 16:11:16,971] INFO : loader split=train rank=0 len=232058 +rank0 [2026-10-27 16:11:16,972] INFO : loader split=valid rank=0 len=48 +rank0 [2026-10-27 16:11:16,972] INFO : Starting training from step 1 to 100000 on rank 0 of 8 +rank0 [2026-10-27 16:11:16,977] INFO : Creating train_iterator +rank0 [2026-10-27 16:11:16,977] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank0 [2026-10-27 16:11:30,841] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank0 [2026-10-27 16:11:45,813] INFO : Created train_iterator +rank0 [2026-10-27 16:12:23,647] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.123548031449318, 'Classification': 0.2298217851272784, 'Regression_pt': 0.7194055180624127, 'Regression_eta': 0.4619348940253258, 'Regression_sin_phi': 0.15005176125559955, 'Regression_cos_phi': 0.14165307599119842, 'Regression_energy': 1.0569261094555258}, weights={'Classification_binary': 1.0, 'Classification': 22.293569564819336, 'Regression_pt': 7.12191915512085, 'Regression_eta': 11.091493606567383, 'Regression_sin_phi': 34.14520263671875, 'Regression_cos_phi': 36.16969299316406, 'Regression_energy': 4.847593307495117} +rank0 [2026-10-27 16:14:49,131] INFO : Step 1000/100000 rank0 | Train Loss: 15.9738 | Train Loss Unweighted: 5.5021 | Train Loss Components: Classification: 0.0191 | Classification_binary: 4.1436 | Regression_cos_phi: 0.0516 | Regression_energy: 0.6035 | Regression_eta: 0.0819 | Regression_pt: 0.5500 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 5.2846 | physical_nopu: 6.1663 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3892 | Classification_binary: 4.2185 | Regression_cos_phi: 2.0475 | Regression_energy: 2.9225 | Regression_eta: 1.0935 | Regression_pt: 3.6578 | Regression_sin_phi: 1.5361 | LR: 4.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.5402s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0013 | max_forward: 0.0194 | max_loss: 0.0039 | max_backward: 0.0308 | max_optimizer_step: 0.1391 | max_tensorboard_logging: 1.0816 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0030 | Memory: rss_MB: 4094.8 | uss_MB: 2885.6 | child_rss_MB: 10815.8 | allocated_MB: 395.5 | reserved_MB: 11198.0 | inactive_split_MB: 104.9 +rank0 [2026-10-27 16:14:49,372] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 100% +4 8 0x7408, 13395 49.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 16% 100% +6 10 0x7408, 62036 45.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:17:31,762] INFO : Step 2000/100000 rank0 | Train Loss: 17.2752 | Train Loss Unweighted: 5.6831 | Train Loss Components: Classification: 0.0157 | Classification_binary: 4.0887 | Regression_cos_phi: 0.0506 | Regression_energy: 0.7236 | Regression_eta: 0.0696 | Regression_pt: 0.6798 | Regression_sin_phi: 0.0552 | Group Train Loss Unweighted (window mean): physical_pu: 4.4021 | physical_nopu: 5.4797 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3426 | Classification_binary: 4.1158 | Regression_cos_phi: 1.4816 | Regression_energy: 3.5071 | Regression_eta: 0.6602 | Regression_pt: 4.8668 | Regression_sin_phi: 1.5478 | LR: 4.00e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.3210s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0250 | max_loss: 0.0036 | max_backward: 0.0657 | max_optimizer_step: 0.1397 | max_tensorboard_logging: 0.8949 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4097.7 | uss_MB: 2895.6 | child_rss_MB: 10834.7 | allocated_MB: 394.2 | reserved_MB: 11198.0 | inactive_split_MB: 90.1 +rank0 [2026-10-27 16:17:31,999] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +2 6 0x7408, 15961 48.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +4 8 0x7408, 13395 50.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 16% 100% +6 10 0x7408, 62036 45.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:20:20,643] INFO : Step 3000/100000 rank0 | Train Loss: 14.0295 | Train Loss Unweighted: 5.1258 | Train Loss Components: Classification: 0.0148 | Classification_binary: 4.0405 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4749 | Regression_eta: 0.0746 | Regression_pt: 0.4205 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 4.1913 | physical_nopu: 5.3261 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2374 | Classification_binary: 4.1752 | Regression_cos_phi: 1.4909 | Regression_energy: 2.2560 | Regression_eta: 0.7232 | Regression_pt: 2.9575 | Regression_sin_phi: 1.5260 | LR: 3.99e-04 | DataLoad Time: 0.0028s | Model Forward Time: 3.1473s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1067 | max_loss: 0.0036 | max_backward: 0.0365 | max_optimizer_step: 0.1415 | max_tensorboard_logging: 1.1098 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0093 | Memory: rss_MB: 4102.6 | uss_MB: 2907.4 | child_rss_MB: 10837.4 | allocated_MB: 395.4 | reserved_MB: 11784.0 | inactive_split_MB: 88.5 +rank0 [2026-10-27 16:20:20,898] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 48.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 16% 100% +6 10 0x7408, 62036 45.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:23:08,741] INFO : Step 4000/100000 rank0 | Train Loss: 11.7244 | Train Loss Unweighted: 3.9925 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.1718 | Regression_cos_phi: 0.0487 | Regression_energy: 0.3212 | Regression_eta: 0.1125 | Regression_pt: 0.2729 | Regression_sin_phi: 0.0492 | Group Train Loss Unweighted (window mean): physical_pu: 4.0838 | physical_nopu: 5.2370 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3489 | Classification_binary: 3.1385 | Regression_cos_phi: 1.6809 | Regression_energy: 1.4878 | Regression_eta: 1.1359 | Regression_pt: 1.8841 | Regression_sin_phi: 1.5461 | LR: 3.99e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.9569s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0037 | max_forward: 0.0726 | max_loss: 0.0066 | max_backward: 0.1670 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4108.6 | uss_MB: 2925.8 | child_rss_MB: 10838.2 | allocated_MB: 406.4 | reserved_MB: 11904.0 | inactive_split_MB: 179.6 +rank0 [2026-10-27 16:23:08,997] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 49.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 16% 100% +6 10 0x7408, 62036 45.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:25:56,681] INFO : Step 5000/100000 rank0 | Train Loss: 13.1313 | Train Loss Unweighted: 4.9481 | Train Loss Components: Classification: 0.0138 | Classification_binary: 4.0144 | Regression_cos_phi: 0.0518 | Regression_energy: 0.3902 | Regression_eta: 0.0928 | Regression_pt: 0.3380 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 4.0057 | physical_nopu: 5.1582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3092 | Classification_binary: 4.0663 | Regression_cos_phi: 1.9440 | Regression_energy: 1.9361 | Regression_eta: 1.3098 | Regression_pt: 2.4680 | Regression_sin_phi: 1.4860 | LR: 3.98e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.0517s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0196 | max_loss: 0.0037 | max_backward: 0.0324 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 1.3896 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0068 | Memory: rss_MB: 4091.1 | uss_MB: 2927.1 | child_rss_MB: 10673.2 | allocated_MB: 393.9 | reserved_MB: 11904.0 | inactive_split_MB: 90.0 +rank0 [2026-10-27 16:25:56,931] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 20% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 132.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 20% 9% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 23% +4 8 0x7408, 13395 49.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 16% 99% +6 10 0x7408, 62036 45.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:28:59,012] INFO : Step 6000/100000 rank0 | Train Loss: 14.5226 | Train Loss Unweighted: 5.0639 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.8601 | Regression_cos_phi: 0.0525 | Regression_energy: 0.5359 | Regression_eta: 0.0694 | Regression_pt: 0.4825 | Regression_sin_phi: 0.0459 | Group Train Loss Unweighted (window mean): physical_pu: 3.9666 | physical_nopu: 5.0829 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4432 | Classification_binary: 4.0557 | Regression_cos_phi: 2.4141 | Regression_energy: 2.2394 | Regression_eta: 0.8299 | Regression_pt: 2.7487 | Regression_sin_phi: 2.2358 | LR: 3.97e-04 | DataLoad Time: 0.0027s | Model Forward Time: 4.2464s | Phase Times: max_pre_step_barrier: 0.0280 | max_batch_to_device: 0.0008 | max_forward: 0.0311 | max_loss: 0.0037 | max_backward: 0.0629 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 2.6604 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0119 | Memory: rss_MB: 4078.1 | uss_MB: 2934.4 | child_rss_MB: 10495.3 | allocated_MB: 394.6 | reserved_MB: 12524.0 | inactive_split_MB: 109.0 +rank0 [2026-10-27 16:28:59,265] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 21% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 45.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:31:52,552] INFO : Step 7000/100000 rank0 | Train Loss: 13.9772 | Train Loss Unweighted: 5.0398 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.9636 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4671 | Regression_eta: 0.0797 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0537 | Group Train Loss Unweighted (window mean): physical_pu: 3.9390 | physical_nopu: 5.0316 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2831 | Classification_binary: 3.7808 | Regression_cos_phi: 1.7643 | Regression_energy: 3.1200 | Regression_eta: 1.1202 | Regression_pt: 4.2917 | Regression_sin_phi: 2.2262 | LR: 3.96e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.8533s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0243 | max_loss: 0.0036 | max_backward: 0.0295 | max_optimizer_step: 0.2429 | max_tensorboard_logging: 2.1406 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 4070.6 | uss_MB: 2937.8 | child_rss_MB: 10370.1 | allocated_MB: 393.9 | reserved_MB: 12524.0 | inactive_split_MB: 107.5 +rank0 [2026-10-27 16:31:52,805] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 21% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 45.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:35:03,719] INFO : Step 8000/100000 rank0 | Train Loss: 13.6161 | Train Loss Unweighted: 4.9650 | Train Loss Components: Classification: 0.0187 | Classification_binary: 3.9275 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4519 | Regression_eta: 0.0615 | Regression_pt: 0.4056 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.9031 | physical_nopu: 4.9878 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3314 | Classification_binary: 3.9194 | Regression_cos_phi: 1.8466 | Regression_energy: 2.1332 | Regression_eta: 0.7686 | Regression_pt: 2.8518 | Regression_sin_phi: 1.8356 | LR: 3.94e-04 | DataLoad Time: 0.0684s | Model Forward Time: 3.5203s | Phase Times: max_pre_step_barrier: 0.0692 | max_batch_to_device: 0.0893 | max_forward: 0.1169 | max_loss: 0.0037 | max_backward: 0.0488 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 1.9012 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4069.6 | uss_MB: 2945.4 | child_rss_MB: 10376.1 | allocated_MB: 393.8 | reserved_MB: 12524.0 | inactive_split_MB: 105.1 +rank0 [2026-10-27 16:35:03,972] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 21% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +2 6 0x7408, 15961 47.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 20% 10% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +6 10 0x7408, 62036 44.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:38:08,203] INFO : Step 9000/100000 rank0 | Train Loss: 15.1606 | Train Loss Unweighted: 5.0855 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.7493 | Regression_cos_phi: 0.0474 | Regression_energy: 0.5909 | Regression_eta: 0.0749 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.8927 | physical_nopu: 4.9579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3132 | Classification_binary: 3.9420 | Regression_cos_phi: 1.4042 | Regression_energy: 2.3935 | Regression_eta: 0.8192 | Regression_pt: 3.1817 | Regression_sin_phi: 1.5830 | LR: 3.93e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.5738s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0007 | max_forward: 0.0386 | max_loss: 0.0041 | max_backward: 0.0582 | max_optimizer_step: 0.1496 | max_tensorboard_logging: 1.9225 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 4056.6 | uss_MB: 2948.9 | child_rss_MB: 10378.8 | allocated_MB: 395.8 | reserved_MB: 12524.0 | inactive_split_MB: 89.5 +rank0 [2026-10-27 16:38:08,462] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 120.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 21% 5% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 19% 2% +2 6 0x7408, 15961 47.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 69% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +6 10 0x7408, 62036 45.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:41:18,990] INFO : Step 10000/100000 rank0 | Train Loss: 12.2525 | Train Loss Unweighted: 4.6725 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8667 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3374 | Regression_eta: 0.0750 | Regression_pt: 0.2772 | Regression_sin_phi: 0.0525 | Group Train Loss Unweighted (window mean): physical_pu: 3.8762 | physical_nopu: 4.9262 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2453 | Classification_binary: 3.9393 | Regression_cos_phi: 2.0171 | Regression_energy: 1.5253 | Regression_eta: 0.7706 | Regression_pt: 1.9619 | Regression_sin_phi: 1.7773 | LR: 3.91e-04 | DataLoad Time: 0.0014s | Model Forward Time: 3.8670s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0277 | max_loss: 0.0035 | max_backward: 0.0304 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 2.2107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4024.3 | uss_MB: 2953.8 | child_rss_MB: 10382.4 | allocated_MB: 393.6 | reserved_MB: 12524.0 | inactive_split_MB: 89.8 +rank0 [2026-10-27 16:41:19,250] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 21% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 49.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +6 10 0x7408, 62036 44.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:41:19,289] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/checkpoints/checkpoint-10000.pth +rank0 [2026-10-27 16:41:19,619] INFO : Running validation on rank0 +rank0 [2026-10-27 16:41:19,619] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 16:41:19,619] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 16:41:19,619] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 16:41:19,619] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 16:41:25,369] INFO : saved batch0_epoch10000.parquet +rank0 [2026-10-27 16:41:25,622] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 16:41:25,729] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 16:41:25,800] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 16:41:25,870] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 16:41:25,929] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 16:41:26,002] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 16:41:26,075] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 16:41:26,139] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 16:41:26,252] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 16:41:26,362] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 16:41:26,437] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 16:41:26,513] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 16:41:26,577] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 16:41:26,667] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 16:41:26,745] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 16:41:26,815] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 16:41:26,910] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 16:41:27,004] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 16:41:27,076] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 16:41:27,149] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 16:41:27,210] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 16:41:27,283] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 16:41:27,357] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 16:41:27,421] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 16:41:27,526] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 16:41:27,619] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 16:41:27,689] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 16:41:27,759] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 16:41:27,819] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 16:41:27,898] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 16:41:27,969] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 16:41:28,032] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 16:41:28,135] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 16:41:28,240] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 16:41:28,315] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 16:41:28,389] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 16:41:28,457] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 16:41:28,528] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 16:41:28,604] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 16:41:28,668] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 16:41:29,234] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 16:41:29,333] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 16:41:29,404] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 16:41:29,472] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 16:41:29,534] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 16:41:29,605] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 16:41:29,679] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 16:41:29,748] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 16:41:29,839] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 16:41:29,933] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 16:41:30,004] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 16:41:30,074] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 16:41:30,135] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 16:41:30,209] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 16:41:30,285] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 16:41:30,350] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 16:41:30,452] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 16:41:30,540] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 16:41:30,612] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 16:41:30,682] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 16:41:30,743] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 16:41:30,816] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 16:41:30,887] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 16:41:30,950] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 16:41:30,953] INFO : plotted pt_target histogram +rank0 [2026-10-27 16:41:30,956] INFO : plotted pt_pred histogram +rank0 [2026-10-27 16:41:30,960] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 16:41:30,963] INFO : plotted eta_target histogram +rank0 [2026-10-27 16:41:30,966] INFO : plotted eta_pred histogram +rank0 [2026-10-27 16:41:30,969] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 16:41:30,972] INFO : plotted sphi_target histogram +rank0 [2026-10-27 16:41:30,973] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 16:41:30,976] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 16:41:30,979] INFO : plotted cphi_target histogram +rank0 [2026-10-27 16:41:30,982] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 16:41:30,985] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 16:41:30,986] INFO : plotted energy_target histogram +rank0 [2026-10-27 16:41:30,989] INFO : plotted energy_pred histogram +rank0 [2026-10-27 16:41:30,992] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 16:43:25,082] INFO : VALIDATION | Step=10000/100000 | Train Loss=12.2525 | Valid Loss=12.5648 | Train Loss Unweighted=4.6725 | Valid Loss Unweighted=4.3786 | Group Valid Loss Unweighted: physical_pu: 3.8471 | physical_nopu: 4.9100 | Stale=0 | ETA=289.2m +rank0 [2026-10-27 16:43:25,091] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:43:25,244] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 16:43:25,271] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 16:43:52,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 16:43:53,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 16:43:53,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 16:43:54,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 16:43:54,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 16:43:55,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 16:43:55,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 16:43:56,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 16:44:06,522] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-27 16:44:06,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:44:06,937] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 16:44:06,941] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 16:44:22,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 16:44:22,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 16:44:23,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 16:44:23,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 16:44:24,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 16:44:24,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 16:44:25,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 16:44:25,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 16:44:35,776] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 16:44:35,782] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:44:36,274] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 16:44:36,278] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 16:44:51,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 16:44:51,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 16:44:52,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 16:44:52,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 16:44:53,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 16:44:53,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 16:44:54,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 16:44:54,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 16:45:04,774] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 16:45:04,785] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar +rank0 [2026-10-27 16:45:10,172] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar +rank0 [2026-10-27 16:45:11,618] INFO : Plotted number of elements +rank0 [2026-10-27 16:45:13,413] INFO : Plotted jets +rank0 [2026-10-27 16:45:14,943] INFO : Plotted jet ratio +rank0 [2026-10-27 16:45:14,983] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd +rank0 [2026-10-27 16:45:20,229] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd +rank0 [2026-10-27 16:45:21,438] INFO : Plotted number of elements +rank0 [2026-10-27 16:45:23,045] INFO : Plotted jets +rank0 [2026-10-27 16:45:24,119] INFO : Plotted jet ratio +rank0 [2026-10-27 16:45:24,156] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt +rank0 [2026-10-27 16:45:29,302] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt +rank0 [2026-10-27 16:45:30,476] INFO : Plotted number of elements +rank0 [2026-10-27 16:45:32,661] INFO : Plotted jets +rank0 [2026-10-27 16:45:33,900] INFO : Plotted jet ratio +rank0 [2026-10-27 16:48:52,121] INFO : Step 11000/100000 rank0 | Train Loss: 13.5331 | Train Loss Unweighted: 4.8490 | Train Loss Components: Classification: 0.0178 | Classification_binary: 3.8332 | Regression_cos_phi: 0.0514 | Regression_energy: 0.4326 | Regression_eta: 0.0715 | Regression_pt: 0.3906 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.8629 | physical_nopu: 4.8860 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3504 | Classification_binary: 3.8133 | Regression_cos_phi: 2.2457 | Regression_energy: 1.9140 | Regression_eta: 0.9065 | Regression_pt: 2.4741 | Regression_sin_phi: 1.7950 | LR: 3.89e-04 | DataLoad Time: 0.0026s | Model Forward Time: 3.7935s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0010 | max_forward: 0.0938 | max_loss: 0.0036 | max_backward: 0.1077 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.9405 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6748.2 | uss_MB: 5693.6 | child_rss_MB: 10378.2 | allocated_MB: 394.1 | reserved_MB: 11016.0 | inactive_split_MB: 102.9 +rank0 [2026-10-27 16:48:52,372] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 100% +4 8 0x7408, 13395 49.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 44.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:51:47,676] INFO : Step 12000/100000 rank0 | Train Loss: 16.0880 | Train Loss Unweighted: 5.2955 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.7102 | Regression_cos_phi: 0.0409 | Regression_energy: 0.7300 | Regression_eta: 0.0621 | Regression_pt: 0.6963 | Regression_sin_phi: 0.0392 | Group Train Loss Unweighted (window mean): physical_pu: 3.8522 | physical_nopu: 4.8676 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3490 | Classification_binary: 3.7126 | Regression_cos_phi: 2.0319 | Regression_energy: 3.1289 | Regression_eta: 0.7573 | Regression_pt: 4.1419 | Regression_sin_phi: 1.3112 | LR: 3.87e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.6532s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1050 | max_forward: 0.2073 | max_loss: 0.0038 | max_backward: 0.1040 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 1.6163 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6734.2 | uss_MB: 5695.7 | child_rss_MB: 10317.1 | allocated_MB: 394.0 | reserved_MB: 11384.0 | inactive_split_MB: 88.0 +rank0 [2026-10-27 16:51:47,933] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +2 6 0x7408, 15961 47.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 100% +4 8 0x7408, 13395 49.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 45.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:54:47,866] INFO : Step 13000/100000 rank0 | Train Loss: 12.2526 | Train Loss Unweighted: 4.6994 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.8775 | Regression_cos_phi: 0.0480 | Regression_energy: 0.3426 | Regression_eta: 0.0688 | Regression_pt: 0.2970 | Regression_sin_phi: 0.0539 | Group Train Loss Unweighted (window mean): physical_pu: 3.8404 | physical_nopu: 4.8415 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3165 | Classification_binary: 3.8406 | Regression_cos_phi: 1.7990 | Regression_energy: 1.5275 | Regression_eta: 0.8567 | Regression_pt: 1.8025 | Regression_sin_phi: 2.1980 | LR: 3.85e-04 | DataLoad Time: 0.0077s | Model Forward Time: 4.0451s | Phase Times: max_pre_step_barrier: 0.0143 | max_batch_to_device: 0.0010 | max_forward: 0.0740 | max_loss: 0.0036 | max_backward: 0.0639 | max_optimizer_step: 0.2642 | max_tensorboard_logging: 2.1016 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 5345.9 | uss_MB: 4318.0 | child_rss_MB: 10308.5 | allocated_MB: 394.7 | reserved_MB: 12168.0 | inactive_split_MB: 111.5 +rank0 [2026-10-27 16:54:48,117] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +2 6 0x7408, 15961 47.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 100% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +6 10 0x7408, 62036 44.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:57:47,322] INFO : Step 14000/100000 rank0 | Train Loss: 11.1328 | Train Loss Unweighted: 3.8032 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.0595 | Regression_cos_phi: 0.0494 | Regression_energy: 0.2823 | Regression_eta: 0.1077 | Regression_pt: 0.2396 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.8281 | physical_nopu: 4.8178 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3958 | Classification_binary: 3.0714 | Regression_cos_phi: 1.7871 | Regression_energy: 1.3431 | Regression_eta: 1.1316 | Regression_pt: 1.6906 | Regression_sin_phi: 1.6996 | LR: 3.83e-04 | DataLoad Time: 0.0022s | Model Forward Time: 4.4520s | Phase Times: max_pre_step_barrier: 0.1064 | max_batch_to_device: 0.0033 | max_forward: 0.1130 | max_loss: 0.0036 | max_backward: 0.2788 | max_optimizer_step: 0.1663 | max_tensorboard_logging: 2.1758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1099 | Memory: rss_MB: 5335.9 | uss_MB: 4329.8 | child_rss_MB: 10259.4 | allocated_MB: 405.3 | reserved_MB: 12168.0 | inactive_split_MB: 126.3 +rank0 [2026-10-27 16:57:47,574] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 123.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 5% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 95% +2 6 0x7408, 15961 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +4 8 0x7408, 13395 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 45% +6 10 0x7408, 62036 44.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:00:51,439] INFO : Step 15000/100000 rank0 | Train Loss: 12.4223 | Train Loss Unweighted: 4.6301 | Train Loss Components: Classification: 0.0132 | Classification_binary: 3.7975 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3484 | Regression_eta: 0.0765 | Regression_pt: 0.2883 | Regression_sin_phi: 0.0502 | Group Train Loss Unweighted (window mean): physical_pu: 3.8205 | physical_nopu: 4.8025 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2358 | Classification_binary: 3.8128 | Regression_cos_phi: 2.0400 | Regression_energy: 1.4082 | Regression_eta: 1.3013 | Regression_pt: 1.6119 | Regression_sin_phi: 1.9875 | LR: 3.80e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.8547s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0799 | max_loss: 0.0851 | max_backward: 0.1383 | max_optimizer_step: 0.1621 | max_tensorboard_logging: 2.1079 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5320.9 | uss_MB: 4324.4 | child_rss_MB: 10137.7 | allocated_MB: 394.1 | reserved_MB: 12168.0 | inactive_split_MB: 89.6 +rank0 [2026-10-27 17:00:51,701] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +4 8 0x7408, 13395 48.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 17% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +6 10 0x7408, 62036 45.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:04:12,921] INFO : Step 16000/100000 rank0 | Train Loss: 13.1559 | Train Loss Unweighted: 4.6950 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.6917 | Regression_cos_phi: 0.0473 | Regression_energy: 0.4330 | Regression_eta: 0.0771 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.8147 | physical_nopu: 4.7795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3346 | Classification_binary: 3.6334 | Regression_cos_phi: 1.5358 | Regression_energy: 2.2392 | Regression_eta: 0.5434 | Regression_pt: 2.9170 | Regression_sin_phi: 1.9990 | LR: 3.78e-04 | DataLoad Time: 0.0030s | Model Forward Time: 3.9735s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.0598 | max_loss: 0.0036 | max_backward: 0.1881 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 2.1012 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5304.0 | uss_MB: 4326.3 | child_rss_MB: 10027.9 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 88.9 +rank0 [2026-10-27 17:04:13,174] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 19% 6% +4 8 0x7408, 13395 49.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +6 10 0x7408, 62036 44.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:07:15,520] INFO : Step 17000/100000 rank0 | Train Loss: 13.6189 | Train Loss Unweighted: 4.8180 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7348 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4756 | Regression_eta: 0.0731 | Regression_pt: 0.4225 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.8095 | physical_nopu: 4.7681 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3409 | Classification_binary: 3.8662 | Regression_cos_phi: 1.8005 | Regression_energy: 1.4448 | Regression_eta: 0.6161 | Regression_pt: 1.8794 | Regression_sin_phi: 1.7855 | LR: 3.75e-04 | DataLoad Time: 0.0026s | Model Forward Time: 4.7210s | Phase Times: max_pre_step_barrier: 0.6520 | max_batch_to_device: 0.0829 | max_forward: 0.1388 | max_loss: 0.0036 | max_backward: 0.0636 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 2.2240 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5273.0 | uss_MB: 4329.2 | child_rss_MB: 9776.7 | allocated_MB: 394.3 | reserved_MB: 12168.0 | inactive_split_MB: 88.4 +rank0 [2026-10-27 17:07:15,769] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +2 6 0x7408, 15961 47.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +4 8 0x7408, 13395 49.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +6 10 0x7408, 62036 44.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:10:19,916] INFO : Step 18000/100000 rank0 | Train Loss: 12.6032 | Train Loss Unweighted: 4.6266 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7426 | Regression_cos_phi: 0.0507 | Regression_energy: 0.3730 | Regression_eta: 0.0736 | Regression_pt: 0.3195 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7954 | physical_nopu: 4.7518 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3113 | Classification_binary: 3.7177 | Regression_cos_phi: 1.8570 | Regression_energy: 2.2703 | Regression_eta: 0.7866 | Regression_pt: 3.0423 | Regression_sin_phi: 1.3234 | LR: 3.72e-04 | DataLoad Time: 0.0067s | Model Forward Time: 4.8567s | Phase Times: max_pre_step_barrier: 0.0217 | max_batch_to_device: 0.0009 | max_forward: 0.1212 | max_loss: 0.0040 | max_backward: 0.1670 | max_optimizer_step: 0.1610 | max_tensorboard_logging: 3.0399 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5245.9 | uss_MB: 4331.6 | child_rss_MB: 9623.0 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 87.1 +rank0 [2026-10-27 17:10:20,193] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 2% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +2 6 0x7408, 15961 46.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +4 8 0x7408, 13395 48.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +6 10 0x7408, 62036 45.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:13:22,192] INFO : Step 19000/100000 rank0 | Train Loss: 14.6123 | Train Loss Unweighted: 5.0036 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.6735 | Regression_cos_phi: 0.0398 | Regression_energy: 0.6019 | Regression_eta: 0.0612 | Regression_pt: 0.5669 | Regression_sin_phi: 0.0441 | Group Train Loss Unweighted (window mean): physical_pu: 3.7887 | physical_nopu: 4.7426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3988 | Classification_binary: 3.7529 | Regression_cos_phi: 1.4795 | Regression_energy: 2.2770 | Regression_eta: 0.7213 | Regression_pt: 2.9275 | Regression_sin_phi: 1.5501 | LR: 3.69e-04 | DataLoad Time: 0.0023s | Model Forward Time: 4.1541s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1593 | max_loss: 0.0094 | max_backward: 0.0749 | max_optimizer_step: 0.1658 | max_tensorboard_logging: 2.3024 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5218.9 | uss_MB: 4333.7 | child_rss_MB: 9466.4 | allocated_MB: 394.5 | reserved_MB: 12168.0 | inactive_split_MB: 103.1 +rank0 [2026-10-27 17:13:22,449] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +6 10 0x7408, 62036 44.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:16:34,703] INFO : Step 20000/100000 rank0 | Train Loss: 13.1046 | Train Loss Unweighted: 4.7571 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.7799 | Regression_cos_phi: 0.0517 | Regression_energy: 0.4251 | Regression_eta: 0.0654 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0511 | Group Train Loss Unweighted (window mean): physical_pu: 3.7835 | physical_nopu: 4.7206 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2509 | Classification_binary: 3.7975 | Regression_cos_phi: 1.7865 | Regression_energy: 1.6235 | Regression_eta: 0.7629 | Regression_pt: 1.9336 | Regression_sin_phi: 1.6528 | LR: 3.66e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.8892s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0008 | max_forward: 0.1344 | max_loss: 0.0037 | max_backward: 0.0763 | max_optimizer_step: 0.1649 | max_tensorboard_logging: 1.9693 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5216.3 | uss_MB: 4336.2 | child_rss_MB: 9462.6 | allocated_MB: 392.8 | reserved_MB: 12168.0 | inactive_split_MB: 89.4 +rank0 [2026-10-27 17:16:34,960] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +6 10 0x7408, 62036 45.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:16:35,101] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/checkpoints/checkpoint-20000.pth +rank0 [2026-10-27 17:16:35,522] INFO : Running validation on rank0 +rank0 [2026-10-27 17:16:35,523] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 17:16:35,523] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 17:16:35,523] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 17:16:35,523] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 17:16:39,004] INFO : saved batch0_epoch20000.parquet +rank0 [2026-10-27 17:16:39,170] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 17:16:39,316] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 17:16:39,399] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 17:16:39,476] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 17:16:39,552] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 17:16:39,631] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 17:16:39,711] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 17:16:39,834] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 17:16:39,966] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 17:16:40,098] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 17:16:40,175] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 17:16:40,251] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 17:16:40,327] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 17:16:40,407] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 17:16:40,484] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 17:16:40,628] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 17:16:40,749] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 17:16:40,870] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 17:16:40,947] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 17:16:41,023] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 17:16:41,099] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 17:16:41,175] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 17:16:41,252] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 17:16:41,360] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 17:16:41,479] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 17:16:41,598] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 17:16:41,674] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 17:16:41,749] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 17:16:41,823] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 17:16:41,898] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 17:16:41,973] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 17:16:42,094] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 17:16:42,227] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 17:16:42,358] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 17:16:42,437] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 17:16:42,948] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 17:16:43,023] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 17:16:43,100] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 17:16:43,177] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 17:16:43,288] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 17:16:43,419] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 17:16:43,547] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 17:16:43,620] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 17:16:43,693] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 17:16:43,767] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 17:16:43,841] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 17:16:43,916] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 17:16:44,020] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 17:16:44,124] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 17:16:44,228] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 17:16:44,300] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 17:16:44,373] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 17:16:44,445] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 17:16:44,520] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 17:16:44,595] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 17:16:44,705] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 17:16:44,815] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 17:16:44,925] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 17:16:44,999] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 17:16:45,074] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 17:16:45,145] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 17:16:45,219] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 17:16:45,293] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 17:16:45,403] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 17:16:45,409] INFO : plotted pt_target histogram +rank0 [2026-10-27 17:16:45,412] INFO : plotted pt_pred histogram +rank0 [2026-10-27 17:16:45,415] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 17:16:45,418] INFO : plotted eta_target histogram +rank0 [2026-10-27 17:16:45,421] INFO : plotted eta_pred histogram +rank0 [2026-10-27 17:16:45,424] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 17:16:45,426] INFO : plotted sphi_target histogram +rank0 [2026-10-27 17:16:45,429] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 17:16:45,432] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 17:16:45,433] INFO : plotted cphi_target histogram +rank0 [2026-10-27 17:16:45,436] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 17:16:45,439] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 17:16:45,442] INFO : plotted energy_target histogram +rank0 [2026-10-27 17:16:45,445] INFO : plotted energy_pred histogram +rank0 [2026-10-27 17:16:45,448] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 17:18:27,947] INFO : VALIDATION | Step=20000/100000 | Train Loss=13.1046 | Valid Loss=12.2009 | Train Loss Unweighted=4.7571 | Valid Loss Unweighted=4.2440 | Group Valid Loss Unweighted: physical_pu: 3.7689 | physical_nopu: 4.7191 | Stale=0 | ETA=268.7m +rank0 [2026-10-27 17:18:27,957] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 17:18:28,107] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 17:18:28,120] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 17:18:53,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 17:18:54,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 17:18:54,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 17:18:55,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 17:18:55,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 17:18:56,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 17:18:56,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 17:18:57,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 17:19:07,069] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 17:19:07,312] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 17:19:07,733] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 17:19:07,736] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 17:19:23,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 17:19:23,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 17:19:24,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 17:19:24,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 17:19:24,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 17:19:25,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 17:19:25,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 17:19:26,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 17:19:36,419] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 17:19:36,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 17:19:36,813] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 17:19:36,815] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 17:19:52,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 17:19:52,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 17:19:53,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 17:19:53,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 17:19:54,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 17:19:54,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 17:19:55,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 17:19:55,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 17:20:05,470] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 17:20:05,665] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar +rank0 [2026-10-27 17:20:10,592] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar +rank0 [2026-10-27 17:20:11,815] INFO : Plotted number of elements +rank0 [2026-10-27 17:20:13,866] INFO : Plotted jets +rank0 [2026-10-27 17:20:15,062] INFO : Plotted jet ratio +rank0 [2026-10-27 17:20:15,107] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd +rank0 [2026-10-27 17:20:20,266] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd +rank0 [2026-10-27 17:20:21,507] INFO : Plotted number of elements +rank0 [2026-10-27 17:20:23,133] INFO : Plotted jets +rank0 [2026-10-27 17:20:24,211] INFO : Plotted jet ratio +rank0 [2026-10-27 17:20:24,250] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt +rank0 [2026-10-27 17:20:29,365] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt +rank0 [2026-10-27 17:20:30,921] INFO : Plotted number of elements +rank0 [2026-10-27 17:20:32,572] INFO : Plotted jets +rank0 [2026-10-27 17:20:33,732] INFO : Plotted jet ratio +rank0 [2026-10-27 17:23:26,112] INFO : Step 21000/100000 rank0 | Train Loss: 12.4320 | Train Loss Unweighted: 4.5235 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.6041 | Regression_cos_phi: 0.0444 | Regression_energy: 0.3902 | Regression_eta: 0.0812 | Regression_pt: 0.3383 | Regression_sin_phi: 0.0478 | Group Train Loss Unweighted (window mean): physical_pu: 3.7732 | physical_nopu: 4.7052 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4372 | Classification_binary: 3.5354 | Regression_cos_phi: 1.4800 | Regression_energy: 1.5642 | Regression_eta: 0.7619 | Regression_pt: 2.0121 | Regression_sin_phi: 1.7436 | LR: 3.62e-04 | DataLoad Time: 0.0026s | Model Forward Time: 4.4184s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.2105 | max_loss: 0.0056 | max_backward: 0.1347 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 2.3606 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5326.2 | uss_MB: 4454.3 | child_rss_MB: 9450.4 | allocated_MB: 394.6 | reserved_MB: 11062.0 | inactive_split_MB: 89.8 +rank0 [2026-10-27 17:23:26,368] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 46.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +4 8 0x7408, 13395 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +6 10 0x7408, 62036 45.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:26:15,412] INFO : Step 22000/100000 rank0 | Train Loss: 14.5232 | Train Loss Unweighted: 4.8893 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.6034 | Regression_cos_phi: 0.0476 | Regression_energy: 0.5714 | Regression_eta: 0.0738 | Regression_pt: 0.5351 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.7709 | physical_nopu: 4.6938 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3767 | Classification_binary: 3.5059 | Regression_cos_phi: 1.8809 | Regression_energy: 2.8872 | Regression_eta: 0.7382 | Regression_pt: 3.9549 | Regression_sin_phi: 1.4727 | LR: 3.59e-04 | DataLoad Time: 0.0029s | Model Forward Time: 3.3844s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0010 | max_forward: 0.0774 | max_loss: 0.0142 | max_backward: 0.0921 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.4442 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1027 | Memory: rss_MB: 5327.2 | uss_MB: 4458.5 | child_rss_MB: 9447.9 | allocated_MB: 394.2 | reserved_MB: 11488.0 | inactive_split_MB: 86.8 +rank0 [2026-10-27 17:26:15,669] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 122.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 3% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +2 6 0x7408, 15961 47.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 84% +4 8 0x7408, 13395 49.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 20% 0% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +6 10 0x7408, 62036 45.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:29:50,723] INFO : Step 23000/100000 rank0 | Train Loss: 12.2140 | Train Loss Unweighted: 4.5907 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.7888 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3388 | Regression_eta: 0.0762 | Regression_pt: 0.2687 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.7690 | physical_nopu: 4.6810 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1498 | Classification_binary: 3.6873 | Regression_cos_phi: 1.5516 | Regression_energy: 1.4933 | Regression_eta: 0.5361 | Regression_pt: 1.8599 | Regression_sin_phi: 1.7609 | LR: 3.55e-04 | DataLoad Time: 0.7658s | Model Forward Time: 4.0690s | Phase Times: max_pre_step_barrier: 0.7590 | max_batch_to_device: 0.0007 | max_forward: 0.0763 | max_loss: 0.0189 | max_backward: 0.1117 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 2.0913 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5325.2 | uss_MB: 4459.8 | child_rss_MB: 9460.6 | allocated_MB: 394.0 | reserved_MB: 12220.0 | inactive_split_MB: 178.1 +rank0 [2026-10-27 17:29:50,972] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 19% 0% +2 6 0x7408, 15961 47.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +4 8 0x7408, 13395 47.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 99% +6 10 0x7408, 62036 44.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:33:22,874] INFO : Step 24000/100000 rank0 | Train Loss: 15.4786 | Train Loss Unweighted: 4.9908 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.5301 | Regression_cos_phi: 0.0467 | Regression_energy: 0.6682 | Regression_eta: 0.0624 | Regression_pt: 0.6223 | Regression_sin_phi: 0.0450 | Group Train Loss Unweighted (window mean): physical_pu: 3.7564 | physical_nopu: 4.6761 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2732 | Classification_binary: 3.5543 | Regression_cos_phi: 2.4051 | Regression_energy: 3.3701 | Regression_eta: 0.7545 | Regression_pt: 4.6173 | Regression_sin_phi: 1.5386 | LR: 3.51e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.6138s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0719 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1672 | max_tensorboard_logging: 1.8279 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5318.8 | uss_MB: 4462.5 | child_rss_MB: 9451.5 | allocated_MB: 393.8 | reserved_MB: 12220.0 | inactive_split_MB: 105.1 +rank0 [2026-10-27 17:33:23,127] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 5% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +4 8 0x7408, 13395 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 99% +6 10 0x7408, 62036 44.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:36:26,737] INFO : Step 25000/100000 rank0 | Train Loss: 12.1246 | Train Loss Unweighted: 4.4783 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.7007 | Regression_cos_phi: 0.0551 | Regression_energy: 0.3196 | Regression_eta: 0.0694 | Regression_pt: 0.2642 | Regression_sin_phi: 0.0578 | Group Train Loss Unweighted (window mean): physical_pu: 3.7531 | physical_nopu: 4.6582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2794 | Classification_binary: 3.7328 | Regression_cos_phi: 2.3229 | Regression_energy: 1.3171 | Regression_eta: 0.8308 | Regression_pt: 1.4917 | Regression_sin_phi: 1.9423 | LR: 3.47e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.7001s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.1917 | max_loss: 0.0039 | max_backward: 0.1029 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 5304.8 | uss_MB: 4464.1 | child_rss_MB: 9445.8 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 86.4 +rank0 [2026-10-27 17:36:26,988] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 100% +6 10 0x7408, 62036 44.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:39:30,795] INFO : Step 26000/100000 rank0 | Train Loss: 12.6920 | Train Loss Unweighted: 4.5479 | Train Loss Components: Classification: 0.0188 | Classification_binary: 3.6003 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4034 | Regression_eta: 0.0734 | Regression_pt: 0.3564 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.7511 | physical_nopu: 4.6505 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4654 | Classification_binary: 3.6792 | Regression_cos_phi: 1.9353 | Regression_energy: 1.6001 | Regression_eta: 0.9460 | Regression_pt: 1.9803 | Regression_sin_phi: 1.7116 | LR: 3.43e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.7866s | Phase Times: max_pre_step_barrier: 0.0479 | max_batch_to_device: 0.0008 | max_forward: 0.1278 | max_loss: 0.0037 | max_backward: 0.0877 | max_optimizer_step: 0.1868 | max_tensorboard_logging: 1.8908 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5292.2 | uss_MB: 4467.2 | child_rss_MB: 9407.7 | allocated_MB: 395.0 | reserved_MB: 12220.0 | inactive_split_MB: 115.0 +rank0 [2026-10-27 17:39:31,049] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +4 8 0x7408, 13395 49.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 99% +6 10 0x7408, 62036 45.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:42:33,568] INFO : Step 27000/100000 rank0 | Train Loss: 16.2758 | Train Loss Unweighted: 5.1421 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5608 | Regression_cos_phi: 0.0448 | Regression_energy: 0.7180 | Regression_eta: 0.0780 | Regression_pt: 0.6791 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.7443 | physical_nopu: 4.6426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3080 | Classification_binary: 3.5878 | Regression_cos_phi: 1.9050 | Regression_energy: 2.4338 | Regression_eta: 1.1909 | Regression_pt: 3.2375 | Regression_sin_phi: 1.5385 | LR: 3.39e-04 | DataLoad Time: 0.0383s | Model Forward Time: 3.6146s | Phase Times: max_pre_step_barrier: 0.0385 | max_batch_to_device: 0.0009 | max_forward: 0.0708 | max_loss: 0.0038 | max_backward: 0.0914 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.8878 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5283.2 | uss_MB: 4469.8 | child_rss_MB: 9400.6 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 87.9 +rank0 [2026-10-27 17:42:33,823] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 49.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 100% +6 10 0x7408, 62036 45.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:45:38,666] INFO : Step 28000/100000 rank0 | Train Loss: 11.9115 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6175 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3339 | Regression_eta: 0.0761 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.7393 | physical_nopu: 4.6288 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3516 | Classification_binary: 3.7168 | Regression_cos_phi: 1.8526 | Regression_energy: 1.6769 | Regression_eta: 0.9983 | Regression_pt: 2.2337 | Regression_sin_phi: 2.1803 | LR: 3.35e-04 | DataLoad Time: 0.0026s | Model Forward Time: 5.4596s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0008 | max_forward: 0.0886 | max_loss: 0.0038 | max_backward: 0.1193 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 3.6285 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 5274.2 | uss_MB: 4146.2 | child_rss_MB: 9355.1 | allocated_MB: 394.0 | reserved_MB: 12414.0 | inactive_split_MB: 106.5 +rank0 [2026-10-27 17:45:38,921] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 99% +2 6 0x7408, 15961 47.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 19% 0% +4 8 0x7408, 13395 48.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 99% +6 10 0x7408, 62036 44.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 1% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:48:55,420] INFO : Step 29000/100000 rank0 | Train Loss: 13.9221 | Train Loss Unweighted: 4.6615 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.5075 | Regression_cos_phi: 0.0491 | Regression_energy: 0.5008 | Regression_eta: 0.0786 | Regression_pt: 0.4589 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7405 | physical_nopu: 4.6233 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4186 | Classification_binary: 3.5404 | Regression_cos_phi: 1.8327 | Regression_energy: 1.9040 | Regression_eta: 0.9043 | Regression_pt: 2.5994 | Regression_sin_phi: 1.6615 | LR: 3.30e-04 | DataLoad Time: 0.0014s | Model Forward Time: 4.0816s | Phase Times: max_pre_step_barrier: 0.1677 | max_batch_to_device: 0.0009 | max_forward: 0.0754 | max_loss: 0.0038 | max_backward: 0.0825 | max_optimizer_step: 0.1673 | max_tensorboard_logging: 2.1272 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5252.2 | uss_MB: 4453.2 | child_rss_MB: 9260.7 | allocated_MB: 394.9 | reserved_MB: 12414.0 | inactive_split_MB: 107.1 +rank0 [2026-10-27 17:48:55,670] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 100% +2 6 0x7408, 15961 47.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 100% +6 10 0x7408, 62036 45.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 19% 5% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:51:58,523] INFO : Step 30000/100000 rank0 | Train Loss: 12.4988 | Train Loss Unweighted: 4.5639 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.6431 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3981 | Regression_eta: 0.0722 | Regression_pt: 0.3405 | Regression_sin_phi: 0.0498 | Group Train Loss Unweighted (window mean): physical_pu: 3.7342 | physical_nopu: 4.6125 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2769 | Classification_binary: 3.5463 | Regression_cos_phi: 1.6394 | Regression_energy: 1.7147 | Regression_eta: 0.8891 | Regression_pt: 2.0376 | Regression_sin_phi: 1.3375 | LR: 3.26e-04 | DataLoad Time: 0.0026s | Model Forward Time: 3.1452s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0762 | max_loss: 0.0136 | max_backward: 0.0854 | max_optimizer_step: 0.1722 | max_tensorboard_logging: 1.3349 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5242.2 | uss_MB: 4475.0 | child_rss_MB: 9193.8 | allocated_MB: 393.7 | reserved_MB: 12480.0 | inactive_split_MB: 89.7 +rank0 [2026-10-27 17:51:58,778] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 99% +2 6 0x7408, 15961 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 0% +4 8 0x7408, 13395 50.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 99% +6 10 0x7408, 62036 46.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 21% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:51:58,939] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/checkpoints/checkpoint-30000.pth +rank0 [2026-10-27 17:51:59,495] INFO : Running validation on rank0 +rank0 [2026-10-27 17:51:59,495] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 17:51:59,495] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 17:51:59,495] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 17:51:59,496] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 17:52:03,084] INFO : saved batch0_epoch30000.parquet +rank0 [2026-10-27 17:52:03,235] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 17:52:03,385] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 17:52:03,467] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 17:52:03,548] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 17:52:03,627] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 17:52:03,707] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 17:52:03,787] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 17:52:03,916] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 17:52:04,053] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 17:52:04,189] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 17:52:04,268] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 17:52:04,347] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 17:52:04,426] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 17:52:04,507] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 17:52:04,587] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 17:52:04,702] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 17:52:04,825] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 17:52:04,950] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 17:52:05,030] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 17:52:05,108] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 17:52:05,188] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 17:52:05,267] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 17:52:05,346] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 17:52:05,457] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 17:52:05,566] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 17:52:05,674] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 17:52:05,751] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 17:52:05,830] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 17:52:05,906] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 17:52:05,984] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 17:52:06,060] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 17:52:06,184] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 17:52:06,319] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 17:52:06,454] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 17:52:06,533] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 17:52:06,614] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 17:52:06,694] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 17:52:07,301] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 17:52:07,382] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 17:52:07,498] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 17:52:07,635] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 17:52:07,767] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 17:52:07,840] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 17:52:07,918] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 17:52:07,993] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 17:52:08,074] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 17:52:08,152] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 17:52:08,258] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 17:52:08,369] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 17:52:08,474] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 17:52:08,551] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 17:52:08,628] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 17:52:08,704] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 17:52:08,783] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 17:52:08,861] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 17:52:08,980] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 17:52:09,102] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 17:52:09,217] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 17:52:09,294] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 17:52:09,371] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 17:52:09,446] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 17:52:09,523] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 17:52:09,600] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 17:52:09,725] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 17:52:09,732] INFO : plotted pt_target histogram +rank0 [2026-10-27 17:52:09,735] INFO : plotted pt_pred histogram +rank0 [2026-10-27 17:52:09,738] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 17:52:09,741] INFO : plotted eta_target histogram +rank0 [2026-10-27 17:52:09,744] INFO : plotted eta_pred histogram +rank0 [2026-10-27 17:52:09,746] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 17:52:09,748] INFO : plotted sphi_target histogram +rank0 [2026-10-27 17:52:09,750] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 17:52:09,753] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 17:52:09,756] INFO : plotted cphi_target histogram +rank0 [2026-10-27 17:52:09,757] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 17:52:09,760] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 17:52:09,762] INFO : plotted energy_target histogram +rank0 [2026-10-27 17:52:09,765] INFO : plotted energy_pred histogram +rank0 [2026-10-27 17:52:09,768] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 17:53:53,742] INFO : VALIDATION | Step=30000/100000 | Train Loss=12.4988 | Valid Loss=12.0271 | Train Loss Unweighted=4.5639 | Valid Loss Unweighted=4.1682 | Group Valid Loss Unweighted: physical_pu: 3.7149 | physical_nopu: 4.6216 | Stale=0 | ETA=239.4m +rank0 [2026-10-27 17:53:53,750] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 17:53:53,901] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 17:53:53,915] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 17:54:18,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 17:54:18,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 17:54:19,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 17:54:19,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 17:54:20,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 17:54:20,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 17:54:21,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 17:54:21,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 17:54:31,723] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 17:54:31,840] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 17:54:32,213] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 17:54:32,216] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 17:54:47,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 17:54:47,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 17:54:48,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 17:54:48,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 17:54:49,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 17:54:49,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 17:54:49,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 17:54:50,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 17:55:00,469] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 17:55:00,637] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 17:55:01,040] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 17:55:01,044] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 17:55:16,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 17:55:17,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 17:55:17,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 17:55:18,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 17:55:18,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 17:55:18,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 17:55:19,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 17:55:19,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 17:55:29,818] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 17:55:30,273] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar +rank0 [2026-10-27 17:55:35,195] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar +rank0 [2026-10-27 17:55:36,304] INFO : Plotted number of elements +rank0 [2026-10-27 17:55:38,387] INFO : Plotted jets +rank0 [2026-10-27 17:55:39,617] INFO : Plotted jet ratio +rank0 [2026-10-27 17:55:39,691] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd +rank0 [2026-10-27 17:55:44,838] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd +rank0 [2026-10-27 17:55:46,152] INFO : Plotted number of elements +rank0 [2026-10-27 17:55:47,826] INFO : Plotted jets +rank0 [2026-10-27 17:55:48,940] INFO : Plotted jet ratio +rank0 [2026-10-27 17:55:48,979] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt +rank0 [2026-10-27 17:55:54,126] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt +rank0 [2026-10-27 17:55:55,329] INFO : Plotted number of elements +rank0 [2026-10-27 17:55:57,420] INFO : Plotted jets +rank0 [2026-10-27 17:55:58,581] INFO : Plotted jet ratio +rank0 [2026-10-27 17:58:48,105] INFO : Step 31000/100000 rank0 | Train Loss: 12.0027 | Train Loss Unweighted: 4.4503 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.5779 | Regression_cos_phi: 0.0460 | Regression_energy: 0.3731 | Regression_eta: 0.0704 | Regression_pt: 0.3230 | Regression_sin_phi: 0.0451 | Group Train Loss Unweighted (window mean): physical_pu: 3.7262 | physical_nopu: 4.6083 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3175 | Classification_binary: 3.6109 | Regression_cos_phi: 1.6060 | Regression_energy: 1.4403 | Regression_eta: 0.6325 | Regression_pt: 1.7384 | Regression_sin_phi: 1.3330 | LR: 3.21e-04 | DataLoad Time: 0.0007s | Model Forward Time: 4.4043s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.1458 | max_loss: 0.0037 | max_backward: 0.1256 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 2.3030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7018.1 | uss_MB: 6249.8 | child_rss_MB: 9197.9 | allocated_MB: 394.8 | reserved_MB: 11168.0 | inactive_split_MB: 96.2 +rank0 [2026-10-27 17:58:48,363] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 17% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 100% +2 6 0x7408, 15961 47.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 45.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:01:55,593] INFO : Step 32000/100000 rank0 | Train Loss: 13.5217 | Train Loss Unweighted: 4.6918 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.5357 | Regression_cos_phi: 0.0443 | Regression_energy: 0.5169 | Regression_eta: 0.0728 | Regression_pt: 0.4649 | Regression_sin_phi: 0.0409 | Group Train Loss Unweighted (window mean): physical_pu: 3.7264 | physical_nopu: 4.6006 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3316 | Classification_binary: 3.4776 | Regression_cos_phi: 1.3049 | Regression_energy: 2.5061 | Regression_eta: 0.7958 | Regression_pt: 3.4568 | Regression_sin_phi: 1.3689 | LR: 3.16e-04 | DataLoad Time: 0.0064s | Model Forward Time: 3.0498s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.0795 | max_loss: 0.0917 | max_backward: 0.1851 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.1890 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5310.0 | uss_MB: 4562.1 | child_rss_MB: 9157.4 | allocated_MB: 394.9 | reserved_MB: 11240.0 | inactive_split_MB: 92.3 +rank0 [2026-10-27 18:01:55,844] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 17% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 49.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 45.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:04:58,439] INFO : Step 33000/100000 rank0 | Train Loss: 11.0126 | Train Loss Unweighted: 3.7465 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9928 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2905 | Regression_eta: 0.1058 | Regression_pt: 0.2472 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.7276 | physical_nopu: 4.5909 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3330 | Classification_binary: 3.0498 | Regression_cos_phi: 1.7738 | Regression_energy: 1.3297 | Regression_eta: 1.2094 | Regression_pt: 1.6449 | Regression_sin_phi: 1.6496 | LR: 3.12e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.6486s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0027 | max_forward: 0.1127 | max_loss: 0.0150 | max_backward: 0.3396 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.4660 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5295.8 | uss_MB: 4291.2 | child_rss_MB: 9057.9 | allocated_MB: 404.8 | reserved_MB: 11922.0 | inactive_split_MB: 135.4 +rank0 [2026-10-27 18:04:58,693] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 17% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 49.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 45.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 15% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:07:50,804] INFO : Step 34000/100000 rank0 | Train Loss: 14.2400 | Train Loss Unweighted: 4.7634 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5051 | Regression_cos_phi: 0.0450 | Regression_energy: 0.5627 | Regression_eta: 0.0631 | Regression_pt: 0.5251 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7180 | physical_nopu: 4.5858 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2796 | Classification_binary: 3.5956 | Regression_cos_phi: 1.3463 | Regression_energy: 2.5313 | Regression_eta: 0.6622 | Regression_pt: 3.4159 | Regression_sin_phi: 1.3130 | LR: 3.07e-04 | DataLoad Time: 0.0015s | Model Forward Time: 3.4214s | Phase Times: max_pre_step_barrier: 0.1802 | max_batch_to_device: 0.0765 | max_forward: 0.1650 | max_loss: 0.0036 | max_backward: 0.0935 | max_optimizer_step: 0.2788 | max_tensorboard_logging: 1.2621 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5285.5 | uss_MB: 4563.5 | child_rss_MB: 9059.3 | allocated_MB: 395.5 | reserved_MB: 11922.0 | inactive_split_MB: 99.3 +rank0 [2026-10-27 18:07:51,060] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 17% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +2 6 0x7408, 15961 47.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +4 8 0x7408, 13395 49.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +6 10 0x7408, 62036 45.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:10:54,319] INFO : Step 35000/100000 rank0 | Train Loss: 11.4005 | Train Loss Unweighted: 4.4142 | Train Loss Components: Classification: 0.0091 | Classification_binary: 3.6972 | Regression_cos_phi: 0.0530 | Regression_energy: 0.2984 | Regression_eta: 0.0671 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.7147 | physical_nopu: 4.5802 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1876 | Classification_binary: 3.9293 | Regression_cos_phi: 1.9288 | Regression_energy: 1.4300 | Regression_eta: 0.6979 | Regression_pt: 1.6587 | Regression_sin_phi: 1.5480 | LR: 3.02e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.0951s | Phase Times: max_pre_step_barrier: 0.0682 | max_batch_to_device: 0.0008 | max_forward: 0.1243 | max_loss: 0.0038 | max_backward: 0.0883 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.1949 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5274.5 | uss_MB: 4566.4 | child_rss_MB: 9063.3 | allocated_MB: 393.8 | reserved_MB: 11922.0 | inactive_split_MB: 85.2 +rank0 [2026-10-27 18:10:54,580] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 17% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 49.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 44.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:13:51,275] INFO : Step 36000/100000 rank0 | Train Loss: 11.8438 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.5751 | Regression_cos_phi: 0.0431 | Regression_energy: 0.3555 | Regression_eta: 0.0641 | Regression_pt: 0.3210 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.7110 | physical_nopu: 4.5729 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4647 | Classification_binary: 3.6272 | Regression_cos_phi: 1.4782 | Regression_energy: 2.0354 | Regression_eta: 0.7641 | Regression_pt: 2.5933 | Regression_sin_phi: 1.8367 | LR: 2.97e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.1457s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0993 | max_loss: 0.0066 | max_backward: 0.1760 | max_optimizer_step: 0.1735 | max_tensorboard_logging: 1.2166 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 5259.5 | uss_MB: 4568.3 | child_rss_MB: 9032.9 | allocated_MB: 395.0 | reserved_MB: 11922.0 | inactive_split_MB: 95.2 +rank0 [2026-10-27 18:13:51,529] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 17% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +2 6 0x7408, 15961 47.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 45.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:16:46,245] INFO : Step 37000/100000 rank0 | Train Loss: 15.1390 | Train Loss Unweighted: 4.8166 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4396 | Regression_cos_phi: 0.0489 | Regression_energy: 0.6142 | Regression_eta: 0.0886 | Regression_pt: 0.5621 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7102 | physical_nopu: 4.5673 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3403 | Classification_binary: 3.6037 | Regression_cos_phi: 2.3927 | Regression_energy: 1.8907 | Regression_eta: 1.0790 | Regression_pt: 2.2024 | Regression_sin_phi: 1.9207 | LR: 2.91e-04 | DataLoad Time: 0.0014s | Model Forward Time: 2.9251s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1889 | max_loss: 0.0036 | max_backward: 0.0899 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 1.0901 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 5243.2 | uss_MB: 4569.9 | child_rss_MB: 9022.4 | allocated_MB: 394.3 | reserved_MB: 11922.0 | inactive_split_MB: 107.1 +rank0 [2026-10-27 18:16:46,508] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +2 6 0x7408, 15961 47.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 49.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 44.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:19:42,758] INFO : Step 38000/100000 rank0 | Train Loss: 11.6256 | Train Loss Unweighted: 4.4424 | Train Loss Components: Classification: 0.0117 | Classification_binary: 3.6501 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3369 | Regression_eta: 0.0664 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7131 | physical_nopu: 4.5682 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2038 | Classification_binary: 3.5794 | Regression_cos_phi: 1.8796 | Regression_energy: 1.6088 | Regression_eta: 1.0173 | Regression_pt: 2.2362 | Regression_sin_phi: 1.7171 | LR: 2.86e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.9294s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.0965 | max_loss: 0.0038 | max_backward: 0.1051 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5237.2 | uss_MB: 4571.0 | child_rss_MB: 9030.3 | allocated_MB: 393.3 | reserved_MB: 12082.0 | inactive_split_MB: 110.5 +rank0 [2026-10-27 18:19:43,010] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +2 6 0x7408, 15961 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 49.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 45.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:22:38,488] INFO : Step 39000/100000 rank0 | Train Loss: 13.0208 | Train Loss Unweighted: 4.4928 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4758 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4360 | Regression_eta: 0.0793 | Regression_pt: 0.3879 | Regression_sin_phi: 0.0500 | Group Train Loss Unweighted (window mean): physical_pu: 3.7060 | physical_nopu: 4.5607 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4034 | Classification_binary: 3.5382 | Regression_cos_phi: 2.2075 | Regression_energy: 2.4345 | Regression_eta: 0.8661 | Regression_pt: 2.9316 | Regression_sin_phi: 2.0698 | LR: 2.81e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.9045s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.0864 | max_loss: 0.0038 | max_backward: 0.0877 | max_optimizer_step: 0.1700 | max_tensorboard_logging: 1.1418 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5224.7 | uss_MB: 4572.8 | child_rss_MB: 9008.2 | allocated_MB: 394.4 | reserved_MB: 12706.0 | inactive_split_MB: 84.9 +rank0 [2026-10-27 18:22:38,747] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +2 6 0x7408, 15961 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 50.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +6 10 0x7408, 62036 46.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:25:31,973] INFO : Step 40000/100000 rank0 | Train Loss: 14.7562 | Train Loss Unweighted: 4.9007 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5708 | Regression_cos_phi: 0.0459 | Regression_energy: 0.5954 | Regression_eta: 0.0747 | Regression_pt: 0.5553 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7006 | physical_nopu: 4.5579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2350 | Classification_binary: 3.7049 | Regression_cos_phi: 1.5802 | Regression_energy: 1.8620 | Regression_eta: 0.8186 | Regression_pt: 2.3209 | Regression_sin_phi: 1.6823 | LR: 2.76e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.9999s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.0814 | max_loss: 0.0614 | max_backward: 0.1417 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 1.1378 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5222.7 | uss_MB: 4573.6 | child_rss_MB: 8992.8 | allocated_MB: 393.7 | reserved_MB: 12706.0 | inactive_split_MB: 107.7 +rank0 [2026-10-27 18:25:32,242] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +2 6 0x7408, 15961 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 50.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +6 10 0x7408, 62036 45.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:25:32,479] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/checkpoints/checkpoint-40000.pth +rank0 [2026-10-27 18:25:32,749] INFO : Running validation on rank0 +rank0 [2026-10-27 18:25:32,749] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 18:25:32,749] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 18:25:32,749] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 18:25:32,749] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 18:25:36,131] INFO : saved batch0_epoch40000.parquet +rank0 [2026-10-27 18:25:36,273] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 18:25:36,413] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 18:25:36,490] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 18:25:36,565] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 18:25:36,640] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 18:25:36,717] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 18:25:36,794] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 18:25:36,916] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 18:25:37,045] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 18:25:37,176] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 18:25:37,254] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 18:25:37,331] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 18:25:37,408] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 18:25:37,485] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 18:25:37,562] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 18:25:37,679] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 18:25:37,801] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 18:25:37,918] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 18:25:37,995] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 18:25:38,071] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 18:25:38,145] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 18:25:38,221] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 18:25:38,299] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 18:25:38,412] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 18:25:38,532] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 18:25:38,651] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 18:25:38,727] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 18:25:38,799] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 18:25:38,871] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 18:25:38,944] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 18:25:39,018] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 18:25:39,140] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 18:25:39,260] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 18:25:39,380] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 18:25:39,459] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 18:25:39,538] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 18:25:40,083] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 18:25:40,159] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 18:25:40,235] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 18:25:40,349] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 18:25:40,480] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 18:25:40,612] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 18:25:40,687] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 18:25:40,764] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 18:25:40,840] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 18:25:40,916] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 18:25:40,992] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 18:25:41,120] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 18:25:41,227] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 18:25:41,332] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 18:25:41,407] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 18:25:41,481] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 18:25:41,555] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 18:25:41,629] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 18:25:41,704] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 18:25:41,818] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 18:25:41,933] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 18:25:42,045] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 18:25:42,120] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 18:25:42,194] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 18:25:42,267] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 18:25:42,343] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 18:25:42,419] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 18:25:42,526] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 18:25:42,532] INFO : plotted pt_target histogram +rank0 [2026-10-27 18:25:42,535] INFO : plotted pt_pred histogram +rank0 [2026-10-27 18:25:42,538] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 18:25:42,541] INFO : plotted eta_target histogram +rank0 [2026-10-27 18:25:42,544] INFO : plotted eta_pred histogram +rank0 [2026-10-27 18:25:42,547] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 18:25:42,548] INFO : plotted sphi_target histogram +rank0 [2026-10-27 18:25:42,550] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 18:25:42,553] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 18:25:42,556] INFO : plotted cphi_target histogram +rank0 [2026-10-27 18:25:42,559] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 18:25:42,562] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 18:25:42,565] INFO : plotted energy_target histogram +rank0 [2026-10-27 18:25:42,568] INFO : plotted energy_pred histogram +rank0 [2026-10-27 18:25:42,572] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 18:27:24,292] INFO : VALIDATION | Step=40000/100000 | Train Loss=14.7562 | Valid Loss=11.9397 | Train Loss Unweighted=4.9007 | Valid Loss Unweighted=4.1367 | Group Valid Loss Unweighted: physical_pu: 3.7005 | physical_nopu: 4.5729 | Stale=0 | ETA=204.2m +rank0 [2026-10-27 18:27:24,301] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 18:27:24,451] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 18:27:24,465] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 18:27:48,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 18:27:48,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 18:27:49,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 18:27:49,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 18:27:50,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 18:27:50,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 18:27:51,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 18:27:51,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 18:28:01,635] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 18:28:01,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 18:28:02,237] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 18:28:02,241] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 18:28:17,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 18:28:18,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 18:28:18,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 18:28:19,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 18:28:19,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 18:28:19,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 18:28:20,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 18:28:20,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 18:28:30,955] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 18:28:30,988] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 18:28:31,367] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 18:28:31,371] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 18:28:46,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 18:28:47,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 18:28:47,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 18:28:48,397] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 18:28:48,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 18:28:49,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 18:28:49,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 18:28:50,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 18:29:00,262] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 18:29:00,306] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar +rank0 [2026-10-27 18:29:05,344] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar +rank0 [2026-10-27 18:29:06,549] INFO : Plotted number of elements +rank0 [2026-10-27 18:29:08,598] INFO : Plotted jets +rank0 [2026-10-27 18:29:09,802] INFO : Plotted jet ratio +rank0 [2026-10-27 18:29:09,869] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd +rank0 [2026-10-27 18:29:14,930] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd +rank0 [2026-10-27 18:29:16,187] INFO : Plotted number of elements +rank0 [2026-10-27 18:29:17,850] INFO : Plotted jets +rank0 [2026-10-27 18:29:18,991] INFO : Plotted jet ratio +rank0 [2026-10-27 18:29:19,028] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt +rank0 [2026-10-27 18:29:24,159] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt +rank0 [2026-10-27 18:29:25,351] INFO : Plotted number of elements +rank0 [2026-10-27 18:29:27,418] INFO : Plotted jets +rank0 [2026-10-27 18:29:28,581] INFO : Plotted jet ratio +rank0 [2026-10-27 18:32:28,109] INFO : Step 41000/100000 rank0 | Train Loss: 12.3609 | Train Loss Unweighted: 4.4817 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5838 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3837 | Regression_eta: 0.0731 | Regression_pt: 0.3287 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.7016 | physical_nopu: 4.5433 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3151 | Classification_binary: 3.6141 | Regression_cos_phi: 1.7246 | Regression_energy: 2.6283 | Regression_eta: 0.6510 | Regression_pt: 3.3615 | Regression_sin_phi: 1.8245 | LR: 2.70e-04 | DataLoad Time: 0.0023s | Model Forward Time: 5.1814s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0009 | max_forward: 0.1338 | max_loss: 0.0038 | max_backward: 0.1857 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 3.1120 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 7028.8 | uss_MB: 6392.1 | child_rss_MB: 8998.3 | allocated_MB: 394.0 | reserved_MB: 11364.0 | inactive_split_MB: 106.1 +rank0 [2026-10-27 18:32:28,381] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 17% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 99% +2 6 0x7408, 15961 47.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 99% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 98% +4 8 0x7408, 13395 48.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +6 10 0x7408, 62036 44.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:35:20,670] INFO : Step 42000/100000 rank0 | Train Loss: 12.9790 | Train Loss Unweighted: 4.5354 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4379 | Regression_cos_phi: 0.0458 | Regression_energy: 0.4856 | Regression_eta: 0.0625 | Regression_pt: 0.4500 | Regression_sin_phi: 0.0370 | Group Train Loss Unweighted (window mean): physical_pu: 3.6959 | physical_nopu: 4.5474 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4101 | Classification_binary: 3.3503 | Regression_cos_phi: 1.6344 | Regression_energy: 3.1013 | Regression_eta: 0.7841 | Regression_pt: 4.3537 | Regression_sin_phi: 1.0275 | LR: 2.65e-04 | DataLoad Time: 0.0127s | Model Forward Time: 3.4409s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0009 | max_forward: 0.1375 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 1.3492 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 7016.8 | uss_MB: 5332.4 | child_rss_MB: 8994.4 | allocated_MB: 394.2 | reserved_MB: 12150.0 | inactive_split_MB: 99.3 +rank0 [2026-10-27 18:35:20,938] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +2 6 0x7408, 15961 47.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +4 8 0x7408, 13395 47.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 99% +6 10 0x7408, 62036 44.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:38:36,611] INFO : Step 43000/100000 rank0 | Train Loss: 10.9006 | Train Loss Unweighted: 3.7057 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.9740 | Regression_cos_phi: 0.0473 | Regression_energy: 0.2786 | Regression_eta: 0.1094 | Regression_pt: 0.2322 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6942 | physical_nopu: 4.5421 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3835 | Classification_binary: 3.0531 | Regression_cos_phi: 1.7054 | Regression_energy: 1.3106 | Regression_eta: 1.0744 | Regression_pt: 1.5937 | Regression_sin_phi: 1.5993 | LR: 2.59e-04 | DataLoad Time: 0.0018s | Model Forward Time: 5.9116s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0039 | max_forward: 0.1100 | max_loss: 0.0637 | max_backward: 0.2303 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 3.9250 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5285.0 | uss_MB: 4686.1 | child_rss_MB: 8875.5 | allocated_MB: 406.8 | reserved_MB: 12150.0 | inactive_split_MB: 152.7 +rank0 [2026-10-27 18:38:36,873] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 48.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 100% +6 10 0x7408, 62036 44.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:41:40,467] INFO : Step 44000/100000 rank0 | Train Loss: 14.8629 | Train Loss Unweighted: 4.8399 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4856 | Regression_cos_phi: 0.0438 | Regression_energy: 0.6070 | Regression_eta: 0.0739 | Regression_pt: 0.5644 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6933 | physical_nopu: 4.5331 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4835 | Classification_binary: 3.3352 | Regression_cos_phi: 1.8500 | Regression_energy: 4.6188 | Regression_eta: 1.1063 | Regression_pt: 6.0806 | Regression_sin_phi: 1.9353 | LR: 2.54e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.3564s | Phase Times: max_pre_step_barrier: 0.0851 | max_batch_to_device: 0.0008 | max_forward: 0.0946 | max_loss: 0.0041 | max_backward: 0.0964 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.3558 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5284.0 | uss_MB: 4679.1 | child_rss_MB: 8885.5 | allocated_MB: 394.8 | reserved_MB: 12150.0 | inactive_split_MB: 94.9 +rank0 [2026-10-27 18:41:40,722] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 100% +6 10 0x7408, 62036 44.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:44:47,640] INFO : Step 45000/100000 rank0 | Train Loss: 12.9535 | Train Loss Unweighted: 4.5658 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.6115 | Regression_cos_phi: 0.0537 | Regression_energy: 0.4117 | Regression_eta: 0.0846 | Regression_pt: 0.3413 | Regression_sin_phi: 0.0531 | Group Train Loss Unweighted (window mean): physical_pu: 3.6892 | physical_nopu: 4.5283 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1993 | Classification_binary: 3.5149 | Regression_cos_phi: 1.7622 | Regression_energy: 2.0717 | Regression_eta: 0.7987 | Regression_pt: 2.2730 | Regression_sin_phi: 1.8221 | LR: 2.48e-04 | DataLoad Time: 0.0035s | Model Forward Time: 3.0789s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1077 | max_loss: 0.0040 | max_backward: 0.1457 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 1.2404 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 5275.4 | uss_MB: 4680.1 | child_rss_MB: 8893.2 | allocated_MB: 394.0 | reserved_MB: 13366.0 | inactive_split_MB: 74.2 +rank0 [2026-10-27 18:44:47,896] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 100% +6 10 0x7408, 62036 44.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 22% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:48:06,964] INFO : Step 46000/100000 rank0 | Train Loss: 12.3783 | Train Loss Unweighted: 4.4476 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.5252 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3923 | Regression_eta: 0.0679 | Regression_pt: 0.3506 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6869 | physical_nopu: 4.5281 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3489 | Classification_binary: 3.6423 | Regression_cos_phi: 1.6315 | Regression_energy: 1.9638 | Regression_eta: 0.8606 | Regression_pt: 2.6890 | Regression_sin_phi: 1.6342 | LR: 2.43e-04 | DataLoad Time: 0.0013s | Model Forward Time: 3.1575s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1021 | max_loss: 0.0036 | max_backward: 0.1895 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 1.1894 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 5269.4 | uss_MB: 4681.7 | child_rss_MB: 8897.5 | allocated_MB: 395.1 | reserved_MB: 13366.0 | inactive_split_MB: 103.1 +rank0 [2026-10-27 18:48:07,224] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 99% +6 10 0x7408, 62036 45.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 22% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:51:16,781] INFO : Step 47000/100000 rank0 | Train Loss: 14.5142 | Train Loss Unweighted: 4.7631 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.3922 | Regression_cos_phi: 0.0407 | Regression_energy: 0.6220 | Regression_eta: 0.0662 | Regression_pt: 0.5862 | Regression_sin_phi: 0.0406 | Group Train Loss Unweighted (window mean): physical_pu: 3.6853 | physical_nopu: 4.5234 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3188 | Classification_binary: 3.6697 | Regression_cos_phi: 1.2591 | Regression_energy: 3.1663 | Regression_eta: 0.7553 | Regression_pt: 4.4920 | Regression_sin_phi: 1.4616 | LR: 2.37e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.1741s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0667 | max_forward: 0.1426 | max_loss: 0.0037 | max_backward: 0.0873 | max_optimizer_step: 0.2591 | max_tensorboard_logging: 1.1650 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5267.4 | uss_MB: 4682.3 | child_rss_MB: 8896.3 | allocated_MB: 394.9 | reserved_MB: 14166.0 | inactive_split_MB: 95.2 +rank0 [2026-10-27 18:51:17,037] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 100% +6 10 0x7408, 62036 45.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 24% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:54:13,527] INFO : Step 48000/100000 rank0 | Train Loss: 11.2030 | Train Loss Unweighted: 4.3343 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.6447 | Regression_cos_phi: 0.0521 | Regression_energy: 0.2822 | Regression_eta: 0.0657 | Regression_pt: 0.2293 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6844 | physical_nopu: 4.5168 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2069 | Classification_binary: 3.5945 | Regression_cos_phi: 2.0430 | Regression_energy: 1.2549 | Regression_eta: 0.6409 | Regression_pt: 1.5516 | Regression_sin_phi: 1.5041 | LR: 2.31e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.9144s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.0058 | max_backward: 0.0852 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1613 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5253.4 | uss_MB: 4683.3 | child_rss_MB: 8890.9 | allocated_MB: 393.9 | reserved_MB: 14166.0 | inactive_split_MB: 101.2 +rank0 [2026-10-27 18:54:13,784] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +2 6 0x7408, 15961 47.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 22% 99% +6 10 0x7408, 62036 45.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 24% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:57:20,593] INFO : Step 49000/100000 rank0 | Train Loss: 12.4372 | Train Loss Unweighted: 4.4483 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.5320 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3881 | Regression_eta: 0.0742 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6801 | physical_nopu: 4.5174 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3245 | Classification_binary: 3.7299 | Regression_cos_phi: 1.7171 | Regression_energy: 1.6256 | Regression_eta: 0.9215 | Regression_pt: 2.1093 | Regression_sin_phi: 1.6989 | LR: 2.26e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.2811s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0009 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1014 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 1.3340 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5240.2 | uss_MB: 4683.4 | child_rss_MB: 8815.5 | allocated_MB: 393.7 | reserved_MB: 14166.0 | inactive_split_MB: 100.8 +rank0 [2026-10-27 18:57:20,877] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 48.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +4 8 0x7408, 13395 49.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 21% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 99% +6 10 0x7408, 62036 45.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 24% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:00:32,917] INFO : Step 50000/100000 rank0 | Train Loss: 13.3654 | Train Loss Unweighted: 4.5742 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4424 | Regression_cos_phi: 0.0426 | Regression_energy: 0.5013 | Regression_eta: 0.0683 | Regression_pt: 0.4573 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6814 | physical_nopu: 4.5132 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3793 | Classification_binary: 3.2748 | Regression_cos_phi: 1.6365 | Regression_energy: 2.1441 | Regression_eta: 0.7932 | Regression_pt: 2.8355 | Regression_sin_phi: 1.4006 | LR: 2.20e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.2672s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0010 | max_forward: 0.1892 | max_loss: 0.0115 | max_backward: 0.1333 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.2641 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0862 | Memory: rss_MB: 5234.2 | uss_MB: 4685.5 | child_rss_MB: 8779.3 | allocated_MB: 394.6 | reserved_MB: 14166.0 | inactive_split_MB: 91.2 +rank0 [2026-10-27 19:00:33,172] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 49.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 21% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +6 10 0x7408, 62036 44.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 24% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:00:33,390] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/checkpoints/checkpoint-50000.pth +rank0 [2026-10-27 19:00:33,936] INFO : Running validation on rank0 +rank0 [2026-10-27 19:00:33,936] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 19:00:33,936] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 19:00:33,936] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 19:00:33,936] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 19:00:37,230] INFO : saved batch0_epoch50000.parquet +rank0 [2026-10-27 19:00:37,383] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 19:00:37,529] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 19:00:37,609] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 19:00:37,684] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 19:00:37,757] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 19:00:37,834] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 19:00:37,911] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 19:00:38,039] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 19:00:38,177] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 19:00:38,314] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 19:00:38,391] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 19:00:38,469] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 19:00:38,546] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 19:00:38,625] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 19:00:38,703] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 19:00:38,822] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 19:00:38,951] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 19:00:39,071] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 19:00:39,150] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 19:00:39,229] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 19:00:39,306] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 19:00:39,385] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 19:00:39,464] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 19:00:39,573] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 19:00:39,699] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 19:00:39,817] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 19:00:39,895] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 19:00:39,971] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 19:00:40,045] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 19:00:40,121] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 19:00:40,197] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 19:00:40,308] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 19:00:40,447] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 19:00:40,578] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 19:00:40,660] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 19:00:40,738] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 19:00:40,814] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 19:00:40,891] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 19:00:41,429] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 19:00:41,546] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 19:00:41,680] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 19:00:41,812] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 19:00:41,890] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 19:00:41,967] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 19:00:42,043] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 19:00:42,120] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 19:00:42,196] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 19:00:42,305] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 19:00:42,414] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 19:00:42,521] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 19:00:42,598] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 19:00:42,672] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 19:00:42,747] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 19:00:42,822] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 19:00:42,897] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 19:00:43,012] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 19:00:43,127] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 19:00:43,241] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 19:00:43,316] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 19:00:43,393] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 19:00:43,468] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 19:00:43,543] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 19:00:43,618] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 19:00:43,748] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 19:00:43,755] INFO : plotted pt_target histogram +rank0 [2026-10-27 19:00:43,758] INFO : plotted pt_pred histogram +rank0 [2026-10-27 19:00:43,764] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 19:00:43,767] INFO : plotted eta_target histogram +rank0 [2026-10-27 19:00:43,769] INFO : plotted eta_pred histogram +rank0 [2026-10-27 19:00:43,772] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 19:00:43,775] INFO : plotted sphi_target histogram +rank0 [2026-10-27 19:00:43,778] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 19:00:43,781] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 19:00:43,784] INFO : plotted cphi_target histogram +rank0 [2026-10-27 19:00:43,786] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 19:00:43,789] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 19:00:43,792] INFO : plotted energy_target histogram +rank0 [2026-10-27 19:00:43,795] INFO : plotted energy_pred histogram +rank0 [2026-10-27 19:00:43,798] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 19:02:38,050] INFO : VALIDATION | Step=50000/100000 | Train Loss=13.3654 | Valid Loss=11.8389 | Train Loss Unweighted=4.5742 | Valid Loss Unweighted=4.0989 | Group Valid Loss Unweighted: physical_pu: 3.6713 | physical_nopu: 4.5265 | Stale=0 | ETA=171.4m +rank0 [2026-10-27 19:02:38,059] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 19:02:38,216] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 19:02:38,230] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 19:03:02,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 19:03:03,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 19:03:03,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 19:03:04,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 19:03:04,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 19:03:05,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 19:03:05,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 19:03:06,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 19:03:16,255] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 19:03:16,261] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 19:03:16,635] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 19:03:16,639] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 19:03:32,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 19:03:32,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 19:03:33,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 19:03:33,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 19:03:33,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 19:03:34,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 19:03:34,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 19:03:35,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 19:03:45,431] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 19:03:45,438] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 19:03:45,816] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 19:03:45,819] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 19:04:00,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 19:04:01,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 19:04:01,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 19:04:02,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 19:04:02,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 19:04:03,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 19:04:03,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 19:04:04,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 19:04:14,298] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 19:04:14,655] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar +rank0 [2026-10-27 19:04:19,955] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar +rank0 [2026-10-27 19:04:21,157] INFO : Plotted number of elements +rank0 [2026-10-27 19:04:23,320] INFO : Plotted jets +rank0 [2026-10-27 19:04:24,590] INFO : Plotted jet ratio +rank0 [2026-10-27 19:04:24,653] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd +rank0 [2026-10-27 19:04:29,988] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd +rank0 [2026-10-27 19:04:31,225] INFO : Plotted number of elements +rank0 [2026-10-27 19:04:32,866] INFO : Plotted jets +rank0 [2026-10-27 19:04:33,982] INFO : Plotted jet ratio +rank0 [2026-10-27 19:04:34,020] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt +rank0 [2026-10-27 19:04:39,178] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt +rank0 [2026-10-27 19:04:40,366] INFO : Plotted number of elements +rank0 [2026-10-27 19:04:42,436] INFO : Plotted jets +rank0 [2026-10-27 19:04:43,592] INFO : Plotted jet ratio +rank0 [2026-10-27 19:07:45,112] INFO : Step 51000/100000 rank0 | Train Loss: 12.0993 | Train Loss Unweighted: 4.3946 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5485 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3591 | Regression_eta: 0.0789 | Regression_pt: 0.2941 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6745 | physical_nopu: 4.5037 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2918 | Classification_binary: 3.6257 | Regression_cos_phi: 1.7338 | Regression_energy: 2.2300 | Regression_eta: 0.8376 | Regression_pt: 2.2946 | Regression_sin_phi: 1.9857 | LR: 2.14e-04 | DataLoad Time: 0.0099s | Model Forward Time: 4.0835s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0008 | max_forward: 0.1391 | max_loss: 0.0036 | max_backward: 0.1509 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 2.0683 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 7102.6 | uss_MB: 6555.6 | child_rss_MB: 8775.1 | allocated_MB: 393.6 | reserved_MB: 10952.0 | inactive_split_MB: 101.2 +rank0 [2026-10-27 19:07:45,369] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +2 6 0x7408, 15961 47.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 48.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 100% +6 10 0x7408, 62036 44.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:10:56,201] INFO : Step 52000/100000 rank0 | Train Loss: 10.7195 | Train Loss Unweighted: 3.6688 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9553 | Regression_cos_phi: 0.0470 | Regression_energy: 0.2719 | Regression_eta: 0.1059 | Regression_pt: 0.2258 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6784 | physical_nopu: 4.5078 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3336 | Classification_binary: 2.9785 | Regression_cos_phi: 1.6610 | Regression_energy: 1.3451 | Regression_eta: 1.1460 | Regression_pt: 1.6242 | Regression_sin_phi: 1.6457 | LR: 2.09e-04 | DataLoad Time: 0.0022s | Model Forward Time: 4.3570s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0031 | max_forward: 0.1853 | max_loss: 0.0256 | max_backward: 0.2604 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 2.0239 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7092.6 | uss_MB: 6564.5 | child_rss_MB: 8773.5 | allocated_MB: 404.8 | reserved_MB: 11626.0 | inactive_split_MB: 256.4 +rank0 [2026-10-27 19:10:56,464] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 46.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 49.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 100% +6 10 0x7408, 62036 45.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:13:49,538] INFO : Step 53000/100000 rank0 | Train Loss: 11.7022 | Train Loss Unweighted: 4.3546 | Train Loss Components: Classification: 0.0123 | Classification_binary: 3.5594 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3321 | Regression_eta: 0.0730 | Regression_pt: 0.2795 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6748 | physical_nopu: 4.5005 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2719 | Classification_binary: 3.6541 | Regression_cos_phi: 2.1584 | Regression_energy: 1.5110 | Regression_eta: 0.9722 | Regression_pt: 1.8662 | Regression_sin_phi: 1.9183 | LR: 2.03e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.8316s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0954 | max_loss: 0.0037 | max_backward: 0.1065 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.9892 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 5255.7 | uss_MB: 4729.4 | child_rss_MB: 8773.6 | allocated_MB: 393.6 | reserved_MB: 11910.0 | inactive_split_MB: 100.6 +rank0 [2026-10-27 19:13:49,794] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 44.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:17:19,414] INFO : Step 54000/100000 rank0 | Train Loss: 12.5762 | Train Loss Unweighted: 4.3665 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0487 | Regression_energy: 0.4104 | Regression_eta: 0.0739 | Regression_pt: 0.3685 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6696 | physical_nopu: 4.5036 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3600 | Classification_binary: 3.5351 | Regression_cos_phi: 2.0058 | Regression_energy: 1.8029 | Regression_eta: 1.1723 | Regression_pt: 2.2751 | Regression_sin_phi: 1.8856 | LR: 1.97e-04 | DataLoad Time: 0.0020s | Model Forward Time: 4.0337s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.1022 | max_forward: 0.1953 | max_loss: 0.0037 | max_backward: 0.0975 | max_optimizer_step: 0.2772 | max_tensorboard_logging: 2.0527 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5248.7 | uss_MB: 4730.3 | child_rss_MB: 8744.8 | allocated_MB: 394.9 | reserved_MB: 11910.0 | inactive_split_MB: 118.2 +rank0 [2026-10-27 19:17:19,675] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 45.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:20:28,879] INFO : Step 55000/100000 rank0 | Train Loss: 13.4632 | Train Loss Unweighted: 4.6500 | Train Loss Components: Classification: 0.0134 | Classification_binary: 3.5428 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4921 | Regression_eta: 0.0679 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6720 | physical_nopu: 4.4919 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2600 | Classification_binary: 3.3940 | Regression_cos_phi: 2.2036 | Regression_energy: 3.2813 | Regression_eta: 0.8794 | Regression_pt: 4.6457 | Regression_sin_phi: 1.5598 | LR: 1.92e-04 | DataLoad Time: 0.0470s | Model Forward Time: 4.3932s | Phase Times: max_pre_step_barrier: 0.1980 | max_batch_to_device: 0.0007 | max_forward: 0.0889 | max_loss: 0.0037 | max_backward: 0.2091 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 2.3779 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5235.8 | uss_MB: 4729.6 | child_rss_MB: 8640.7 | allocated_MB: 393.8 | reserved_MB: 11910.0 | inactive_split_MB: 89.1 +rank0 [2026-10-27 19:20:29,138] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 20% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 44.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:23:31,318] INFO : Step 56000/100000 rank0 | Train Loss: 11.6078 | Train Loss Unweighted: 4.2962 | Train Loss Components: Classification: 0.0140 | Classification_binary: 3.4759 | Regression_cos_phi: 0.0438 | Regression_energy: 0.3484 | Regression_eta: 0.0671 | Regression_pt: 0.2977 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6692 | physical_nopu: 4.4908 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2674 | Classification_binary: 3.7652 | Regression_cos_phi: 2.2091 | Regression_energy: 1.9372 | Regression_eta: 0.9546 | Regression_pt: 2.2783 | Regression_sin_phi: 2.2958 | LR: 1.86e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.9398s | Phase Times: max_pre_step_barrier: 0.0941 | max_batch_to_device: 0.0008 | max_forward: 0.0930 | max_loss: 0.0038 | max_backward: 0.0988 | max_optimizer_step: 0.1756 | max_tensorboard_logging: 1.8920 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 5221.8 | uss_MB: 4732.9 | child_rss_MB: 8582.9 | allocated_MB: 393.5 | reserved_MB: 11910.0 | inactive_split_MB: 115.7 +rank0 [2026-10-27 19:23:31,577] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 20% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 49.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 45.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:26:53,591] INFO : Step 57000/100000 rank0 | Train Loss: 13.9252 | Train Loss Unweighted: 4.6468 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4640 | Regression_cos_phi: 0.0464 | Regression_energy: 0.5144 | Regression_eta: 0.0781 | Regression_pt: 0.4791 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6687 | physical_nopu: 4.4875 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4692 | Classification_binary: 3.2994 | Regression_cos_phi: 1.4812 | Regression_energy: 2.5992 | Regression_eta: 1.1349 | Regression_pt: 3.5651 | Regression_sin_phi: 1.5885 | LR: 1.81e-04 | DataLoad Time: 0.0022s | Model Forward Time: 5.3114s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.1223 | max_forward: 0.2094 | max_loss: 0.0420 | max_backward: 0.1628 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 2.9072 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 5216.8 | uss_MB: 4734.2 | child_rss_MB: 8511.2 | allocated_MB: 394.3 | reserved_MB: 12240.0 | inactive_split_MB: 101.4 +rank0 [2026-10-27 19:26:53,853] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 20% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 100% +2 6 0x7408, 15961 47.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 48.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 44.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:30:08,344] INFO : Step 58000/100000 rank0 | Train Loss: 11.5910 | Train Loss Unweighted: 4.3047 | Train Loss Components: Classification: 0.0106 | Classification_binary: 3.5490 | Regression_cos_phi: 0.0544 | Regression_energy: 0.3108 | Regression_eta: 0.0631 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0510 | Group Train Loss Unweighted (window mean): physical_pu: 3.6630 | physical_nopu: 4.4842 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2801 | Classification_binary: 3.5895 | Regression_cos_phi: 1.9942 | Regression_energy: 1.9064 | Regression_eta: 0.7273 | Regression_pt: 2.2887 | Regression_sin_phi: 1.4808 | LR: 1.75e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.2018s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.1080 | max_backward: 0.1964 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.2749 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 5209.3 | uss_MB: 4734.4 | child_rss_MB: 8432.6 | allocated_MB: 393.9 | reserved_MB: 12240.0 | inactive_split_MB: 121.9 +rank0 [2026-10-27 19:30:08,605] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 20% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 100% +2 6 0x7408, 15961 47.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 45.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:33:12,863] INFO : Step 59000/100000 rank0 | Train Loss: 12.2218 | Train Loss Unweighted: 4.4115 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5166 | Regression_cos_phi: 0.0458 | Regression_energy: 0.3794 | Regression_eta: 0.0771 | Regression_pt: 0.3280 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6650 | physical_nopu: 4.4788 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3542 | Classification_binary: 3.4135 | Regression_cos_phi: 1.6204 | Regression_energy: 2.2075 | Regression_eta: 0.8749 | Regression_pt: 2.9773 | Regression_sin_phi: 1.4276 | LR: 1.70e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.9314s | Phase Times: max_pre_step_barrier: 0.6889 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0499 | max_backward: 0.1549 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5201.3 | uss_MB: 4735.4 | child_rss_MB: 8387.3 | allocated_MB: 394.0 | reserved_MB: 12240.0 | inactive_split_MB: 135.3 +rank0 [2026-10-27 19:33:13,125] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 5% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 99% +2 6 0x7408, 15961 47.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +4 8 0x7408, 13395 49.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +6 10 0x7408, 62036 45.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:36:27,208] INFO : Step 60000/100000 rank0 | Train Loss: 14.3810 | Train Loss Unweighted: 4.7229 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.4146 | Regression_cos_phi: 0.0449 | Regression_energy: 0.5841 | Regression_eta: 0.0716 | Regression_pt: 0.5495 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.6604 | physical_nopu: 4.4865 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3476 | Classification_binary: 3.2836 | Regression_cos_phi: 2.1000 | Regression_energy: 3.5304 | Regression_eta: 0.6625 | Regression_pt: 4.9897 | Regression_sin_phi: 1.7739 | LR: 1.64e-04 | DataLoad Time: 0.0031s | Model Forward Time: 3.1440s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0849 | max_loss: 0.0036 | max_backward: 0.1478 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 1.2880 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5191.9 | uss_MB: 4495.0 | child_rss_MB: 8328.9 | allocated_MB: 394.8 | reserved_MB: 12240.0 | inactive_split_MB: 87.0 +rank0 [2026-10-27 19:36:27,471] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 20% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 21% 99% +2 6 0x7408, 15961 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +4 8 0x7408, 13395 49.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 96% +6 10 0x7408, 62036 45.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:36:27,783] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/checkpoints/checkpoint-60000.pth +rank0 [2026-10-27 19:36:28,857] INFO : Running validation on rank0 +rank0 [2026-10-27 19:36:28,857] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 19:36:28,857] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 19:36:28,857] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 19:36:28,857] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 19:36:32,107] INFO : saved batch0_epoch60000.parquet +rank0 [2026-10-27 19:36:32,273] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 19:36:32,419] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 19:36:32,498] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 19:36:32,575] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 19:36:32,652] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 19:36:32,731] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 19:36:32,808] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 19:36:32,939] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 19:36:33,073] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 19:36:33,205] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 19:36:33,283] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 19:36:33,360] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 19:36:33,438] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 19:36:33,515] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 19:36:33,593] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 19:36:33,716] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 19:36:33,848] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 19:36:33,980] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 19:36:34,059] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 19:36:34,135] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 19:36:34,212] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 19:36:34,290] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 19:36:34,368] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 19:36:34,476] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 19:36:34,604] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 19:36:34,723] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 19:36:34,798] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 19:36:34,880] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 19:36:34,959] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 19:36:35,037] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 19:36:35,113] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 19:36:35,228] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 19:36:35,364] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 19:36:35,502] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 19:36:35,581] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 19:36:35,661] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 19:36:35,740] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 19:36:36,391] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 19:36:36,470] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 19:36:36,588] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 19:36:36,722] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 19:36:36,853] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 19:36:36,932] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 19:36:37,008] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 19:36:37,084] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 19:36:37,163] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 19:36:37,241] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 19:36:37,375] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 19:36:37,488] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 19:36:37,596] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 19:36:37,670] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 19:36:37,747] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 19:36:37,823] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 19:36:37,901] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 19:36:37,978] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 19:36:38,095] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 19:36:38,213] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 19:36:38,329] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 19:36:38,407] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 19:36:38,485] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 19:36:38,562] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 19:36:38,640] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 19:36:38,718] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 19:36:38,830] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 19:36:38,836] INFO : plotted pt_target histogram +rank0 [2026-10-27 19:36:38,839] INFO : plotted pt_pred histogram +rank0 [2026-10-27 19:36:38,842] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 19:36:38,845] INFO : plotted eta_target histogram +rank0 [2026-10-27 19:36:38,848] INFO : plotted eta_pred histogram +rank0 [2026-10-27 19:36:38,851] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 19:36:38,854] INFO : plotted sphi_target histogram +rank0 [2026-10-27 19:36:38,857] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 19:36:38,859] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 19:36:38,862] INFO : plotted cphi_target histogram +rank0 [2026-10-27 19:36:38,865] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 19:36:38,869] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 19:36:38,872] INFO : plotted energy_target histogram +rank0 [2026-10-27 19:36:38,875] INFO : plotted energy_pred histogram +rank0 [2026-10-27 19:36:38,878] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 19:38:19,034] INFO : VALIDATION | Step=60000/100000 | Train Loss=14.3810 | Valid Loss=11.7671 | Train Loss Unweighted=4.7229 | Valid Loss Unweighted=4.0735 | Group Valid Loss Unweighted: physical_pu: 3.6582 | physical_nopu: 4.4889 | Stale=0 | ETA=138.0m +rank0 [2026-10-27 19:38:19,043] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 19:38:19,194] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 19:38:19,206] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 19:38:43,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 19:38:44,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 19:38:44,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 19:38:45,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 19:38:45,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 19:38:46,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 19:38:46,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 19:38:47,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 19:38:57,235] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 19:38:57,240] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 19:38:57,609] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 19:38:57,612] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 19:39:12,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 19:39:12,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 19:39:13,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 19:39:13,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 19:39:14,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 19:39:14,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 19:39:15,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 19:39:15,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 19:39:25,681] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 19:39:26,170] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 19:39:26,533] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 19:39:26,536] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 19:39:41,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 19:39:42,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 19:39:42,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 19:39:42,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 19:39:43,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 19:39:43,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 19:39:44,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 19:39:44,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 19:39:54,922] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 19:39:54,985] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar +rank0 [2026-10-27 19:39:59,892] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar +rank0 [2026-10-27 19:40:01,022] INFO : Plotted number of elements +rank0 [2026-10-27 19:40:03,074] INFO : Plotted jets +rank0 [2026-10-27 19:40:04,245] INFO : Plotted jet ratio +rank0 [2026-10-27 19:40:04,338] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd +rank0 [2026-10-27 19:40:09,262] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd +rank0 [2026-10-27 19:40:10,487] INFO : Plotted number of elements +rank0 [2026-10-27 19:40:12,129] INFO : Plotted jets +rank0 [2026-10-27 19:40:13,255] INFO : Plotted jet ratio +rank0 [2026-10-27 19:40:13,292] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt +rank0 [2026-10-27 19:40:18,245] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt +rank0 [2026-10-27 19:40:19,413] INFO : Plotted number of elements +rank0 [2026-10-27 19:40:21,474] INFO : Plotted jets +rank0 [2026-10-27 19:40:22,637] INFO : Plotted jet ratio +rank0 [2026-10-27 19:43:34,200] INFO : Step 61000/100000 rank0 | Train Loss: 11.4262 | Train Loss Unweighted: 4.3706 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.6500 | Regression_cos_phi: 0.0531 | Regression_energy: 0.2956 | Regression_eta: 0.0675 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6553 | physical_nopu: 4.4780 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2828 | Classification_binary: 3.8456 | Regression_cos_phi: 2.1665 | Regression_energy: 1.4290 | Regression_eta: 0.7748 | Regression_pt: 1.6677 | Regression_sin_phi: 2.1416 | LR: 1.59e-04 | DataLoad Time: 0.0073s | Model Forward Time: 3.8527s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1116 | max_loss: 0.0036 | max_backward: 0.1506 | max_optimizer_step: 0.1828 | max_tensorboard_logging: 1.9738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 5784.0 | uss_MB: 5329.2 | child_rss_MB: 8302.4 | allocated_MB: 393.5 | reserved_MB: 11190.0 | inactive_split_MB: 103.1 +rank0 [2026-10-27 19:43:34,460] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 17% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +2 6 0x7408, 15961 48.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 44.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:46:50,279] INFO : Step 62000/100000 rank0 | Train Loss: 10.7781 | Train Loss Unweighted: 3.6510 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9253 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2791 | Regression_eta: 0.1043 | Regression_pt: 0.2316 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3435 | Classification_binary: 2.9749 | Regression_cos_phi: 1.8642 | Regression_energy: 1.3600 | Regression_eta: 1.2131 | Regression_pt: 1.6138 | Regression_sin_phi: 1.6081 | LR: 1.54e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.3309s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0030 | max_forward: 0.1294 | max_loss: 0.0038 | max_backward: 0.2007 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.2115 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0496 | Memory: rss_MB: 5775.0 | uss_MB: 5052.2 | child_rss_MB: 8186.6 | allocated_MB: 404.4 | reserved_MB: 11190.0 | inactive_split_MB: 148.6 +rank0 [2026-10-27 19:46:50,542] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 17% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 90% +2 6 0x7408, 15961 46.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 18% 6% +6 10 0x7408, 62036 45.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 88% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:50:03,174] INFO : Step 63000/100000 rank0 | Train Loss: 11.5766 | Train Loss Unweighted: 4.3655 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6000 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3204 | Regression_eta: 0.0727 | Regression_pt: 0.2610 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6589 | physical_nopu: 4.4771 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2528 | Classification_binary: 3.6709 | Regression_cos_phi: 1.8301 | Regression_energy: 1.5746 | Regression_eta: 0.8661 | Regression_pt: 1.9104 | Regression_sin_phi: 1.6171 | LR: 1.49e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.1185s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.1036 | max_loss: 0.0037 | max_backward: 0.1140 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2208 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 5773.0 | uss_MB: 5331.6 | child_rss_MB: 8138.8 | allocated_MB: 393.8 | reserved_MB: 11190.0 | inactive_split_MB: 93.0 +rank0 [2026-10-27 19:50:03,430] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 17% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +2 6 0x7408, 15961 47.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 49.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 45.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:53:18,449] INFO : Step 64000/100000 rank0 | Train Loss: 13.2709 | Train Loss Unweighted: 4.5542 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.4295 | Regression_cos_phi: 0.0429 | Regression_energy: 0.4948 | Regression_eta: 0.0672 | Regression_pt: 0.4587 | Regression_sin_phi: 0.0436 | Group Train Loss Unweighted (window mean): physical_pu: 3.6551 | physical_nopu: 4.4714 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3355 | Classification_binary: 3.3099 | Regression_cos_phi: 1.7082 | Regression_energy: 1.5721 | Regression_eta: 0.6146 | Regression_pt: 2.0383 | Regression_sin_phi: 1.3059 | LR: 1.43e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.0707s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0928 | max_loss: 0.0570 | max_backward: 0.1589 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 1.2035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 5289.6 | uss_MB: 4857.6 | child_rss_MB: 8120.4 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank0 [2026-10-27 19:53:18,709] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 44.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:56:23,771] INFO : Step 65000/100000 rank0 | Train Loss: 15.6652 | Train Loss Unweighted: 4.9833 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5112 | Regression_cos_phi: 0.0496 | Regression_energy: 0.6688 | Regression_eta: 0.0629 | Regression_pt: 0.6290 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6554 | physical_nopu: 4.4650 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2698 | Classification_binary: 3.6304 | Regression_cos_phi: 2.1279 | Regression_energy: 1.8656 | Regression_eta: 0.8666 | Regression_pt: 2.3977 | Regression_sin_phi: 1.6536 | LR: 1.38e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.4771s | Phase Times: max_pre_step_barrier: 0.1158 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0042 | max_backward: 0.1974 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.4176 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5282.6 | uss_MB: 4858.3 | child_rss_MB: 8025.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 124.9 +rank0 [2026-10-27 19:56:24,027] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 49.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 46.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:59:29,403] INFO : Step 66000/100000 rank0 | Train Loss: 12.0073 | Train Loss Unweighted: 4.3350 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4949 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3535 | Regression_eta: 0.0691 | Regression_pt: 0.3018 | Regression_sin_phi: 0.0501 | Group Train Loss Unweighted (window mean): physical_pu: 3.6526 | physical_nopu: 4.4657 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2835 | Classification_binary: 3.4805 | Regression_cos_phi: 1.7325 | Regression_energy: 1.7114 | Regression_eta: 1.0366 | Regression_pt: 1.9909 | Regression_sin_phi: 1.5311 | LR: 1.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.1251s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0891 | max_loss: 0.0538 | max_backward: 0.1468 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 1.2753 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5277.7 | uss_MB: 4858.9 | child_rss_MB: 7982.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 107.1 +rank0 [2026-10-27 19:59:29,655] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 50.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 46.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:02:52,635] INFO : Step 67000/100000 rank0 | Train Loss: 13.3177 | Train Loss Unweighted: 4.5554 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.4800 | Regression_cos_phi: 0.0500 | Regression_energy: 0.4690 | Regression_eta: 0.0673 | Regression_pt: 0.4253 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6538 | physical_nopu: 4.4665 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3617 | Classification_binary: 3.3345 | Regression_cos_phi: 1.9328 | Regression_energy: 1.9036 | Regression_eta: 0.7289 | Regression_pt: 2.5655 | Regression_sin_phi: 1.5972 | LR: 1.28e-04 | DataLoad Time: 0.0781s | Model Forward Time: 4.5505s | Phase Times: max_pre_step_barrier: 0.0792 | max_batch_to_device: 0.0009 | max_forward: 0.0886 | max_loss: 0.1057 | max_backward: 0.1987 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 2.6109 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 5271.7 | uss_MB: 4859.8 | child_rss_MB: 7952.8 | allocated_MB: 393.9 | reserved_MB: 11792.0 | inactive_split_MB: 111.5 +rank0 [2026-10-27 20:02:52,891] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 49.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 45.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:06:09,756] INFO : Step 68000/100000 rank0 | Train Loss: 11.1535 | Train Loss Unweighted: 4.2643 | Train Loss Components: Classification: 0.0109 | Classification_binary: 3.5138 | Regression_cos_phi: 0.0456 | Regression_energy: 0.3163 | Regression_eta: 0.0648 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6527 | physical_nopu: 4.4693 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2592 | Classification_binary: 3.4329 | Regression_cos_phi: 1.4414 | Regression_energy: 1.2806 | Regression_eta: 0.5983 | Regression_pt: 1.5740 | Regression_sin_phi: 1.5364 | LR: 1.24e-04 | DataLoad Time: 0.0011s | Model Forward Time: 3.0409s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1008 | max_loss: 0.0767 | max_backward: 0.1768 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.1759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5263.4 | uss_MB: 4860.3 | child_rss_MB: 7883.7 | allocated_MB: 394.5 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank0 [2026-10-27 20:06:10,014] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 49.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +6 10 0x7408, 62036 45.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:09:13,169] INFO : Step 69000/100000 rank0 | Train Loss: 12.1728 | Train Loss Unweighted: 4.3418 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4719 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3660 | Regression_eta: 0.0763 | Regression_pt: 0.3115 | Regression_sin_phi: 0.0548 | Group Train Loss Unweighted (window mean): physical_pu: 3.6494 | physical_nopu: 4.4629 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4239 | Classification_binary: 3.4642 | Regression_cos_phi: 1.7811 | Regression_energy: 2.1081 | Regression_eta: 0.7069 | Regression_pt: 2.5476 | Regression_sin_phi: 1.5988 | LR: 1.19e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.9690s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.1094 | max_loss: 0.0147 | max_backward: 0.1167 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 1.1204 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5251.4 | uss_MB: 4861.5 | child_rss_MB: 7816.8 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 106.5 +rank0 [2026-10-27 20:09:13,424] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 48.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 44.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:12:14,146] INFO : Step 70000/100000 rank0 | Train Loss: 14.9156 | Train Loss Unweighted: 4.7730 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.3975 | Regression_cos_phi: 0.0487 | Regression_energy: 0.6183 | Regression_eta: 0.0638 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0448 | Group Train Loss Unweighted (window mean): physical_pu: 3.6488 | physical_nopu: 4.4584 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3745 | Classification_binary: 3.4142 | Regression_cos_phi: 1.8755 | Regression_energy: 2.7829 | Regression_eta: 0.6630 | Regression_pt: 3.9808 | Regression_sin_phi: 1.4493 | LR: 1.14e-04 | DataLoad Time: 0.0065s | Model Forward Time: 3.0324s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1038 | max_loss: 0.0858 | max_backward: 0.1871 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 1.1619 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5245.8 | uss_MB: 4860.8 | child_rss_MB: 7775.8 | allocated_MB: 395.0 | reserved_MB: 11792.0 | inactive_split_MB: 111.9 +rank0 [2026-10-27 20:12:14,403] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 49.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 44.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:12:14,590] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/checkpoints/checkpoint-70000.pth +rank0 [2026-10-27 20:12:14,871] INFO : Running validation on rank0 +rank0 [2026-10-27 20:12:14,871] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 20:12:14,872] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 20:12:14,872] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 20:12:14,872] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 20:12:18,325] INFO : saved batch0_epoch70000.parquet +rank0 [2026-10-27 20:12:18,484] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 20:12:18,626] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 20:12:18,708] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 20:12:18,785] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 20:12:18,862] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 20:12:18,940] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 20:12:19,018] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 20:12:19,147] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 20:12:19,281] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 20:12:19,414] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 20:12:19,492] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 20:12:19,569] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 20:12:19,644] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 20:12:19,722] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 20:12:19,804] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 20:12:19,931] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 20:12:20,052] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 20:12:20,174] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 20:12:20,254] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 20:12:20,334] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 20:12:20,412] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 20:12:20,492] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 20:12:20,570] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 20:12:20,681] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 20:12:20,805] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 20:12:20,931] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 20:12:21,009] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 20:12:21,087] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 20:12:21,165] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 20:12:21,243] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 20:12:21,319] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 20:12:21,437] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 20:12:21,571] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 20:12:21,706] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 20:12:21,787] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 20:12:21,865] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 20:12:21,942] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 20:12:22,021] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 20:12:22,099] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 20:12:22,765] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 20:12:22,899] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 20:12:23,034] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 20:12:23,115] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 20:12:23,191] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 20:12:23,266] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 20:12:23,342] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 20:12:23,420] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 20:12:23,552] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 20:12:23,662] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 20:12:23,772] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 20:12:23,848] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 20:12:23,925] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 20:12:23,998] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 20:12:24,072] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 20:12:24,151] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 20:12:24,272] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 20:12:24,390] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 20:12:24,503] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 20:12:24,578] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 20:12:24,654] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 20:12:24,730] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 20:12:24,810] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 20:12:24,888] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 20:12:25,000] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 20:12:25,006] INFO : plotted pt_target histogram +rank0 [2026-10-27 20:12:25,009] INFO : plotted pt_pred histogram +rank0 [2026-10-27 20:12:25,012] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 20:12:25,015] INFO : plotted eta_target histogram +rank0 [2026-10-27 20:12:25,018] INFO : plotted eta_pred histogram +rank0 [2026-10-27 20:12:25,021] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 20:12:25,024] INFO : plotted sphi_target histogram +rank0 [2026-10-27 20:12:25,027] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 20:12:25,030] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 20:12:25,032] INFO : plotted cphi_target histogram +rank0 [2026-10-27 20:12:25,035] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 20:12:25,038] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 20:12:25,041] INFO : plotted energy_target histogram +rank0 [2026-10-27 20:12:25,044] INFO : plotted energy_pred histogram +rank0 [2026-10-27 20:12:25,047] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 20:14:09,877] INFO : VALIDATION | Step=70000/100000 | Train Loss=14.9156 | Valid Loss=11.7243 | Train Loss Unweighted=4.7730 | Valid Loss Unweighted=4.0574 | Group Valid Loss Unweighted: physical_pu: 3.6474 | physical_nopu: 4.4674 | Stale=0 | ETA=104.1m +rank0 [2026-10-27 20:14:09,886] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 20:14:10,036] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 20:14:10,040] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 20:14:34,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 20:14:35,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 20:14:35,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 20:14:36,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 20:14:36,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 20:14:37,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 20:14:37,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 20:14:37,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 20:14:47,902] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 20:14:47,942] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 20:14:48,315] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 20:14:48,319] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 20:15:03,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 20:15:03,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 20:15:04,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 20:15:04,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 20:15:05,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 20:15:05,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 20:15:06,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 20:15:06,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 20:15:16,736] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 20:15:17,038] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 20:15:17,438] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 20:15:17,442] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 20:15:32,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 20:15:33,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 20:15:33,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 20:15:34,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 20:15:34,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 20:15:35,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 20:15:35,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 20:15:36,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 20:15:46,133] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 20:15:46,181] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar +rank0 [2026-10-27 20:15:51,012] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar +rank0 [2026-10-27 20:15:52,155] INFO : Plotted number of elements +rank0 [2026-10-27 20:15:54,262] INFO : Plotted jets +rank0 [2026-10-27 20:15:55,485] INFO : Plotted jet ratio +rank0 [2026-10-27 20:15:55,529] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd +rank0 [2026-10-27 20:16:00,403] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd +rank0 [2026-10-27 20:16:01,597] INFO : Plotted number of elements +rank0 [2026-10-27 20:16:03,245] INFO : Plotted jets +rank0 [2026-10-27 20:16:04,339] INFO : Plotted jet ratio +rank0 [2026-10-27 20:16:04,377] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt +rank0 [2026-10-27 20:16:09,355] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt +rank0 [2026-10-27 20:16:10,558] INFO : Plotted number of elements +rank0 [2026-10-27 20:16:12,688] INFO : Plotted jets +rank0 [2026-10-27 20:16:13,884] INFO : Plotted jet ratio +rank0 [2026-10-27 20:19:22,610] INFO : Step 71000/100000 rank0 | Train Loss: 10.9922 | Train Loss Unweighted: 3.6772 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9201 | Regression_cos_phi: 0.0477 | Regression_energy: 0.2906 | Regression_eta: 0.1066 | Regression_pt: 0.2485 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6484 | physical_nopu: 4.4561 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3213 | Classification_binary: 2.9922 | Regression_cos_phi: 1.6891 | Regression_energy: 1.2927 | Regression_eta: 1.0693 | Regression_pt: 1.5585 | Regression_sin_phi: 1.5380 | LR: 1.10e-04 | DataLoad Time: 0.0018s | Model Forward Time: 4.3920s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0652 | max_forward: 0.2596 | max_loss: 0.0036 | max_backward: 0.3497 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 1.8772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 7279.8 | uss_MB: 6900.2 | child_rss_MB: 7804.4 | allocated_MB: 405.9 | reserved_MB: 11130.0 | inactive_split_MB: 135.5 +rank0 [2026-10-27 20:19:22,866] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 16% 100% +2 6 0x7408, 15961 47.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 100% +6 10 0x7408, 62036 44.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:22:35,483] INFO : Step 72000/100000 rank0 | Train Loss: 14.4733 | Train Loss Unweighted: 4.6898 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3667 | Regression_cos_phi: 0.0458 | Regression_energy: 0.5948 | Regression_eta: 0.0650 | Regression_pt: 0.5567 | Regression_sin_phi: 0.0444 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4536 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4221 | Classification_binary: 3.5394 | Regression_cos_phi: 1.8715 | Regression_energy: 1.7957 | Regression_eta: 0.7587 | Regression_pt: 2.4136 | Regression_sin_phi: 2.0251 | LR: 1.05e-04 | DataLoad Time: 0.0010s | Model Forward Time: 3.2103s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1390 | max_loss: 0.0037 | max_backward: 0.1007 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.2476 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1281 | Memory: rss_MB: 5316.2 | uss_MB: 4936.1 | child_rss_MB: 7727.0 | allocated_MB: 393.8 | reserved_MB: 11706.0 | inactive_split_MB: 88.5 +rank0 [2026-10-27 20:22:35,737] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 45.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:25:35,076] INFO : Step 73000/100000 rank0 | Train Loss: 11.7903 | Train Loss Unweighted: 4.2561 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.5010 | Regression_cos_phi: 0.0578 | Regression_energy: 0.3092 | Regression_eta: 0.0644 | Regression_pt: 0.2565 | Regression_sin_phi: 0.0557 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4567 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2562 | Classification_binary: 3.3405 | Regression_cos_phi: 2.0976 | Regression_energy: 1.3503 | Regression_eta: 0.6215 | Regression_pt: 1.7852 | Regression_sin_phi: 1.9634 | LR: 1.01e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.2509s | Phase Times: max_pre_step_barrier: 0.1246 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0122 | max_backward: 0.1972 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.2348 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5307.2 | uss_MB: 4937.3 | child_rss_MB: 7690.0 | allocated_MB: 393.5 | reserved_MB: 11706.0 | inactive_split_MB: 93.2 +rank0 [2026-10-27 20:25:35,332] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +2 6 0x7408, 15961 47.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +6 10 0x7408, 62036 44.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:28:48,015] INFO : Step 74000/100000 rank0 | Train Loss: 11.6748 | Train Loss Unweighted: 4.2803 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4309 | Regression_cos_phi: 0.0428 | Regression_energy: 0.3572 | Regression_eta: 0.0695 | Regression_pt: 0.3177 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.6485 | physical_nopu: 4.4502 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3633 | Classification_binary: 3.1843 | Regression_cos_phi: 1.2433 | Regression_energy: 1.3770 | Regression_eta: 0.5196 | Regression_pt: 1.8811 | Regression_sin_phi: 1.2949 | LR: 9.68e-05 | DataLoad Time: 0.0009s | Model Forward Time: 3.6917s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1195 | max_loss: 0.0060 | max_backward: 0.2082 | max_optimizer_step: 0.1763 | max_tensorboard_logging: 1.8023 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5300.9 | uss_MB: 4938.3 | child_rss_MB: 7614.5 | allocated_MB: 394.3 | reserved_MB: 11706.0 | inactive_split_MB: 97.0 +rank0 [2026-10-27 20:28:48,262] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 45.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:31:57,104] INFO : Step 75000/100000 rank0 | Train Loss: 14.1604 | Train Loss Unweighted: 4.6445 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.4007 | Regression_cos_phi: 0.0489 | Regression_energy: 0.5577 | Regression_eta: 0.0640 | Regression_pt: 0.5112 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.6489 | physical_nopu: 4.4484 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2980 | Classification_binary: 3.0863 | Regression_cos_phi: 1.2621 | Regression_energy: 4.7788 | Regression_eta: 0.4909 | Regression_pt: 7.1309 | Regression_sin_phi: 1.2284 | LR: 9.27e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.5572s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0831 | max_loss: 0.0038 | max_backward: 0.2058 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.5751 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1279 | Memory: rss_MB: 5290.9 | uss_MB: 4938.0 | child_rss_MB: 7581.3 | allocated_MB: 394.4 | reserved_MB: 11706.0 | inactive_split_MB: 89.1 +rank0 [2026-10-27 20:31:57,364] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 45.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 17% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:34:58,400] INFO : Step 76000/100000 rank0 | Train Loss: 11.4149 | Train Loss Unweighted: 4.2984 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5224 | Regression_cos_phi: 0.0478 | Regression_energy: 0.3298 | Regression_eta: 0.0709 | Regression_pt: 0.2685 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6454 | physical_nopu: 4.4486 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3109 | Classification_binary: 3.5237 | Regression_cos_phi: 1.6398 | Regression_energy: 1.6953 | Regression_eta: 0.7124 | Regression_pt: 2.1407 | Regression_sin_phi: 1.4586 | LR: 8.88e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.8038s | Phase Times: max_pre_step_barrier: 0.0296 | max_batch_to_device: 0.0996 | max_forward: 0.1816 | max_loss: 0.0036 | max_backward: 0.0930 | max_optimizer_step: 0.2829 | max_tensorboard_logging: 1.7104 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1287 | Memory: rss_MB: 5286.0 | uss_MB: 4937.3 | child_rss_MB: 7586.8 | allocated_MB: 393.6 | reserved_MB: 12330.0 | inactive_split_MB: 97.7 +rank0 [2026-10-27 20:34:58,654] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 21% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 44.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:38:17,521] INFO : Step 77000/100000 rank0 | Train Loss: 12.8789 | Train Loss Unweighted: 4.4653 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4333 | Regression_cos_phi: 0.0478 | Regression_energy: 0.4517 | Regression_eta: 0.0624 | Regression_pt: 0.4079 | Regression_sin_phi: 0.0461 | Group Train Loss Unweighted (window mean): physical_pu: 3.6433 | physical_nopu: 4.4448 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3829 | Classification_binary: 3.4919 | Regression_cos_phi: 1.6265 | Regression_energy: 2.6379 | Regression_eta: 0.5591 | Regression_pt: 3.6689 | Regression_sin_phi: 1.6881 | LR: 8.50e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.6700s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.0831 | max_loss: 0.0514 | max_backward: 0.1524 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.8652 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5273.0 | uss_MB: 4939.8 | child_rss_MB: 7463.9 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 99.2 +rank0 [2026-10-27 20:38:17,772] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 21% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 49.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 45.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:41:22,362] INFO : Step 78000/100000 rank0 | Train Loss: 13.2420 | Train Loss Unweighted: 4.6345 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.5815 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4598 | Regression_eta: 0.0679 | Regression_pt: 0.4144 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6415 | physical_nopu: 4.4494 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2634 | Classification_binary: 3.5963 | Regression_cos_phi: 1.8977 | Regression_energy: 1.9540 | Regression_eta: 0.6727 | Regression_pt: 2.6243 | Regression_sin_phi: 1.6595 | LR: 8.13e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.7705s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0927 | max_loss: 0.0036 | max_backward: 0.2060 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.8450 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5261.0 | uss_MB: 4940.8 | child_rss_MB: 7383.4 | allocated_MB: 394.7 | reserved_MB: 12330.0 | inactive_split_MB: 95.1 +rank0 [2026-10-27 20:41:22,621] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 21% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 49.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +6 10 0x7408, 62036 44.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:44:32,084] INFO : Step 79000/100000 rank0 | Train Loss: 11.5047 | Train Loss Unweighted: 4.2761 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4919 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3261 | Regression_eta: 0.0640 | Regression_pt: 0.2834 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6416 | physical_nopu: 4.4460 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3082 | Classification_binary: 3.4963 | Regression_cos_phi: 1.6804 | Regression_energy: 1.4670 | Regression_eta: 0.8215 | Regression_pt: 1.9656 | Regression_sin_phi: 1.6459 | LR: 7.78e-05 | DataLoad Time: 0.0026s | Model Forward Time: 3.8920s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0253 | max_forward: 0.1473 | max_loss: 0.0038 | max_backward: 0.0955 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 1.9569 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5248.8 | uss_MB: 4513.6 | child_rss_MB: 7321.5 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 120.0 +rank0 [2026-10-27 20:44:32,339] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 21% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +6 10 0x7408, 62036 45.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:47:44,482] INFO : Step 80000/100000 rank0 | Train Loss: 15.3065 | Train Loss Unweighted: 4.8564 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4208 | Regression_cos_phi: 0.0455 | Regression_energy: 0.6415 | Regression_eta: 0.0773 | Regression_pt: 0.6092 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6398 | physical_nopu: 4.4406 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4095 | Classification_binary: 3.3525 | Regression_cos_phi: 1.3418 | Regression_energy: 2.8622 | Regression_eta: 1.1996 | Regression_pt: 3.9157 | Regression_sin_phi: 1.3498 | LR: 7.44e-05 | DataLoad Time: 0.0069s | Model Forward Time: 3.7266s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0010 | max_forward: 0.0840 | max_loss: 0.0039 | max_backward: 0.0916 | max_optimizer_step: 0.2630 | max_tensorboard_logging: 1.7339 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5238.8 | uss_MB: 4942.0 | child_rss_MB: 7231.1 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 141.7 +rank0 [2026-10-27 20:47:44,746] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 21% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +6 10 0x7408, 62036 44.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:47:44,922] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/checkpoints/checkpoint-80000.pth +rank0 [2026-10-27 20:47:46,091] INFO : Running validation on rank0 +rank0 [2026-10-27 20:47:46,091] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 20:47:46,091] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 20:47:46,091] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 20:47:46,091] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 20:47:49,749] INFO : saved batch0_epoch80000.parquet +rank0 [2026-10-27 20:47:49,888] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 20:47:50,028] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 20:47:50,108] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 20:47:50,184] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 20:47:50,258] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 20:47:50,335] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 20:47:50,411] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 20:47:50,535] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 20:47:50,664] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 20:47:50,794] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 20:47:50,871] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 20:47:50,949] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 20:47:51,025] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 20:47:51,103] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 20:47:51,181] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 20:47:51,307] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 20:47:51,428] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 20:47:51,562] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 20:47:51,642] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 20:47:51,723] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 20:47:51,803] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 20:47:51,886] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 20:47:51,967] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 20:47:52,078] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 20:47:52,202] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 20:47:52,334] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 20:47:52,412] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 20:47:52,492] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 20:47:52,569] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 20:47:52,648] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 20:47:52,727] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 20:47:52,846] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 20:47:52,981] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 20:47:53,120] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 20:47:53,201] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 20:47:53,282] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 20:47:53,362] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 20:47:53,446] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 20:47:54,119] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 20:47:54,241] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 20:47:54,379] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 20:47:54,514] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 20:47:54,591] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 20:47:54,669] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 20:47:54,744] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 20:47:54,825] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 20:47:54,905] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 20:47:55,015] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 20:47:55,127] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 20:47:55,236] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 20:47:55,311] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 20:47:55,387] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 20:47:55,466] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 20:47:55,544] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 20:47:55,626] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 20:47:55,750] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 20:47:55,868] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 20:47:55,987] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 20:47:56,063] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 20:47:56,157] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 20:47:56,233] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 20:47:56,311] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 20:47:56,389] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 20:47:56,504] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 20:47:56,510] INFO : plotted pt_target histogram +rank0 [2026-10-27 20:47:56,513] INFO : plotted pt_pred histogram +rank0 [2026-10-27 20:47:56,518] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 20:47:56,519] INFO : plotted eta_target histogram +rank0 [2026-10-27 20:47:56,522] INFO : plotted eta_pred histogram +rank0 [2026-10-27 20:47:56,525] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 20:47:56,527] INFO : plotted sphi_target histogram +rank0 [2026-10-27 20:47:56,530] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 20:47:56,533] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 20:47:56,536] INFO : plotted cphi_target histogram +rank0 [2026-10-27 20:47:56,539] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 20:47:56,542] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 20:47:56,543] INFO : plotted energy_target histogram +rank0 [2026-10-27 20:47:56,546] INFO : plotted energy_pred histogram +rank0 [2026-10-27 20:47:56,549] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 20:49:38,944] INFO : VALIDATION | Step=80000/100000 | Train Loss=15.3065 | Valid Loss=11.6948 | Train Loss Unweighted=4.8564 | Valid Loss Unweighted=4.0477 | Group Valid Loss Unweighted: physical_pu: 3.6398 | physical_nopu: 4.4556 | Stale=0 | ETA=69.6m +rank0 [2026-10-27 20:49:38,953] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 20:49:39,102] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 20:49:39,106] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 20:50:04,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 20:50:05,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 20:50:05,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 20:50:06,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 20:50:06,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 20:50:07,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 20:50:07,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 20:50:08,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 20:50:18,262] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 20:50:18,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 20:50:18,743] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 20:50:18,747] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 20:50:33,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 20:50:34,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 20:50:34,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 20:50:35,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 20:50:35,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 20:50:35,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 20:50:36,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 20:50:37,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 20:50:47,014] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 20:50:47,266] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 20:50:47,676] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 20:50:47,679] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 20:51:02,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 20:51:03,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 20:51:03,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 20:51:04,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 20:51:04,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 20:51:05,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 20:51:05,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 20:51:06,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 20:51:16,043] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 20:51:16,275] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar +rank0 [2026-10-27 20:51:21,214] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar +rank0 [2026-10-27 20:51:22,417] INFO : Plotted number of elements +rank0 [2026-10-27 20:51:24,539] INFO : Plotted jets +rank0 [2026-10-27 20:51:25,753] INFO : Plotted jet ratio +rank0 [2026-10-27 20:51:25,813] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd +rank0 [2026-10-27 20:51:30,714] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd +rank0 [2026-10-27 20:51:31,913] INFO : Plotted number of elements +rank0 [2026-10-27 20:51:33,552] INFO : Plotted jets +rank0 [2026-10-27 20:51:34,658] INFO : Plotted jet ratio +rank0 [2026-10-27 20:51:34,697] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt +rank0 [2026-10-27 20:51:39,614] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt +rank0 [2026-10-27 20:51:40,792] INFO : Plotted number of elements +rank0 [2026-10-27 20:51:42,874] INFO : Plotted jets +rank0 [2026-10-27 20:51:44,029] INFO : Plotted jet ratio +rank0 [2026-10-27 20:54:48,064] INFO : Step 81000/100000 rank0 | Train Loss: 10.7072 | Train Loss Unweighted: 3.6241 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.9080 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2710 | Regression_eta: 0.1078 | Regression_pt: 0.2269 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6413 | physical_nopu: 4.4395 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3405 | Classification_binary: 2.9291 | Regression_cos_phi: 1.7542 | Regression_energy: 1.3048 | Regression_eta: 1.1777 | Regression_pt: 1.5982 | Regression_sin_phi: 1.5974 | LR: 7.11e-05 | DataLoad Time: 0.0015s | Model Forward Time: 4.1295s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0033 | max_forward: 0.1984 | max_loss: 0.0038 | max_backward: 0.3244 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 1.8991 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 7314.5 | uss_MB: 7027.3 | child_rss_MB: 7215.1 | allocated_MB: 408.0 | reserved_MB: 11650.0 | inactive_split_MB: 261.6 +rank0 [2026-10-27 20:54:48,320] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 17% 100% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 44.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:58:05,391] INFO : Step 82000/100000 rank0 | Train Loss: 12.9094 | Train Loss Unweighted: 4.6043 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5247 | Regression_cos_phi: 0.0417 | Regression_energy: 0.4765 | Regression_eta: 0.0603 | Regression_pt: 0.4449 | Regression_sin_phi: 0.0402 | Group Train Loss Unweighted (window mean): physical_pu: 3.6409 | physical_nopu: 4.4420 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2729 | Classification_binary: 3.7036 | Regression_cos_phi: 1.6888 | Regression_energy: 1.9389 | Regression_eta: 0.7814 | Regression_pt: 2.5018 | Regression_sin_phi: 1.6023 | LR: 6.80e-05 | DataLoad Time: 0.0145s | Model Forward Time: 3.8367s | Phase Times: max_pre_step_barrier: 0.0156 | max_batch_to_device: 0.0008 | max_forward: 0.1472 | max_loss: 0.0038 | max_backward: 0.1553 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 1.8498 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5342.3 | uss_MB: 5059.9 | child_rss_MB: 7159.0 | allocated_MB: 393.8 | reserved_MB: 11650.0 | inactive_split_MB: 92.9 +rank0 [2026-10-27 20:58:05,647] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 44.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:01:21,218] INFO : Step 83000/100000 rank0 | Train Loss: 11.2556 | Train Loss Unweighted: 4.2618 | Train Loss Components: Classification: 0.0094 | Classification_binary: 3.5167 | Regression_cos_phi: 0.0501 | Regression_energy: 0.3127 | Regression_eta: 0.0669 | Regression_pt: 0.2586 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6408 | physical_nopu: 4.4397 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2441 | Classification_binary: 3.6990 | Regression_cos_phi: 1.7837 | Regression_energy: 1.2610 | Regression_eta: 0.8348 | Regression_pt: 1.3550 | Regression_sin_phi: 1.6446 | LR: 6.51e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.2360s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0008 | max_forward: 0.1683 | max_loss: 0.0037 | max_backward: 0.1023 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.3519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 5336.3 | uss_MB: 5061.6 | child_rss_MB: 7095.7 | allocated_MB: 393.2 | reserved_MB: 11650.0 | inactive_split_MB: 95.2 +rank0 [2026-10-27 21:01:21,503] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 44.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:04:34,355] INFO : Step 84000/100000 rank0 | Train Loss: 11.5964 | Train Loss Unweighted: 4.2611 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4346 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3486 | Regression_eta: 0.0638 | Regression_pt: 0.3055 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6366 | physical_nopu: 4.4419 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.5273 | Classification_binary: 3.3008 | Regression_cos_phi: 1.8729 | Regression_energy: 1.8356 | Regression_eta: 0.6472 | Regression_pt: 2.4871 | Regression_sin_phi: 1.4470 | LR: 6.23e-05 | DataLoad Time: 0.0014s | Model Forward Time: 5.6205s | Phase Times: max_pre_step_barrier: 0.2933 | max_batch_to_device: 0.0008 | max_forward: 0.1129 | max_loss: 0.0110 | max_backward: 0.1312 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 3.4426 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5330.3 | uss_MB: 5063.1 | child_rss_MB: 7029.9 | allocated_MB: 393.9 | reserved_MB: 11650.0 | inactive_split_MB: 95.6 +rank0 [2026-10-27 21:04:34,612] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +4 8 0x7408, 13395 48.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 72% +6 10 0x7408, 62036 45.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:07:29,235] INFO : Step 85000/100000 rank0 | Train Loss: 14.1090 | Train Loss Unweighted: 4.6641 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.3775 | Regression_cos_phi: 0.0425 | Regression_energy: 0.5754 | Regression_eta: 0.0615 | Regression_pt: 0.5479 | Regression_sin_phi: 0.0420 | Group Train Loss Unweighted (window mean): physical_pu: 3.6325 | physical_nopu: 4.4336 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3704 | Classification_binary: 3.2605 | Regression_cos_phi: 1.2444 | Regression_energy: 2.1040 | Regression_eta: 0.6208 | Regression_pt: 3.0373 | Regression_sin_phi: 1.1577 | LR: 5.96e-05 | DataLoad Time: 0.0012s | Model Forward Time: 3.2646s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0496 | max_backward: 0.1488 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 1.3583 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 5323.1 | uss_MB: 5063.4 | child_rss_MB: 7037.3 | allocated_MB: 394.0 | reserved_MB: 11698.0 | inactive_split_MB: 93.1 +rank0 [2026-10-27 21:07:29,503] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 49.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 45.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:10:27,476] INFO : Step 86000/100000 rank0 | Train Loss: 11.7921 | Train Loss Unweighted: 4.3412 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.5823 | Regression_cos_phi: 0.0537 | Regression_energy: 0.3118 | Regression_eta: 0.0750 | Regression_pt: 0.2521 | Regression_sin_phi: 0.0550 | Group Train Loss Unweighted (window mean): physical_pu: 3.6404 | physical_nopu: 4.4337 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3093 | Classification_binary: 3.8187 | Regression_cos_phi: 2.0281 | Regression_energy: 1.8865 | Regression_eta: 0.6906 | Regression_pt: 2.2679 | Regression_sin_phi: 2.0847 | LR: 5.71e-05 | DataLoad Time: 0.0229s | Model Forward Time: 3.0825s | Phase Times: max_pre_step_barrier: 0.0233 | max_batch_to_device: 0.0898 | max_forward: 0.1824 | max_loss: 0.0038 | max_backward: 0.0938 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.9995 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1323 | Memory: rss_MB: 5319.1 | uss_MB: 4616.7 | child_rss_MB: 7006.7 | allocated_MB: 393.2 | reserved_MB: 11704.0 | inactive_split_MB: 96.0 +rank0 [2026-10-27 21:10:27,729] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 49.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 45.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:13:31,795] INFO : Step 87000/100000 rank0 | Train Loss: 12.0054 | Train Loss Unweighted: 4.3232 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.3964 | Regression_cos_phi: 0.0444 | Regression_energy: 0.4046 | Regression_eta: 0.0620 | Regression_pt: 0.3567 | Regression_sin_phi: 0.0418 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4455 | Classification_binary: 3.3651 | Regression_cos_phi: 1.6006 | Regression_energy: 1.6940 | Regression_eta: 0.8496 | Regression_pt: 2.2725 | Regression_sin_phi: 1.4526 | LR: 5.48e-05 | DataLoad Time: 0.0020s | Model Forward Time: 2.9469s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.1290 | max_loss: 0.0039 | max_backward: 0.0975 | max_optimizer_step: 0.1784 | max_tensorboard_logging: 1.1620 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5310.9 | uss_MB: 5064.7 | child_rss_MB: 7011.5 | allocated_MB: 394.7 | reserved_MB: 11704.0 | inactive_split_MB: 94.3 +rank0 [2026-10-27 21:13:32,053] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 49.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 45.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:16:34,119] INFO : Step 88000/100000 rank0 | Train Loss: 14.2663 | Train Loss Unweighted: 4.6673 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.4533 | Regression_cos_phi: 0.0523 | Regression_energy: 0.5342 | Regression_eta: 0.0666 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0508 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3953 | Classification_binary: 3.3443 | Regression_cos_phi: 2.2808 | Regression_energy: 1.6063 | Regression_eta: 0.6276 | Regression_pt: 1.8902 | Regression_sin_phi: 1.4094 | LR: 5.26e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.9907s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0008 | max_forward: 0.0997 | max_loss: 0.0039 | max_backward: 0.1038 | max_optimizer_step: 0.2517 | max_tensorboard_logging: 1.0175 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5309.9 | uss_MB: 5065.2 | child_rss_MB: 7012.4 | allocated_MB: 393.7 | reserved_MB: 11704.0 | inactive_split_MB: 103.2 +rank0 [2026-10-27 21:16:34,382] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +2 6 0x7408, 15961 47.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 49.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 45.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:19:31,508] INFO : Step 89000/100000 rank0 | Train Loss: 11.9885 | Train Loss Unweighted: 4.3122 | Train Loss Components: Classification: 0.0138 | Classification_binary: 3.4837 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3438 | Regression_eta: 0.0687 | Regression_pt: 0.2984 | Regression_sin_phi: 0.0554 | Group Train Loss Unweighted (window mean): physical_pu: 3.6329 | physical_nopu: 4.4374 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3203 | Classification_binary: 3.3780 | Regression_cos_phi: 1.8408 | Regression_energy: 1.5262 | Regression_eta: 0.6267 | Regression_pt: 2.0842 | Regression_sin_phi: 1.9794 | LR: 5.06e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.9310s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0976 | max_loss: 0.0053 | max_backward: 0.1242 | max_optimizer_step: 0.1785 | max_tensorboard_logging: 1.0326 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 5307.5 | uss_MB: 5066.4 | child_rss_MB: 7006.7 | allocated_MB: 395.0 | reserved_MB: 13422.0 | inactive_split_MB: 109.9 +rank0 [2026-10-27 21:19:31,774] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 1% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +2 6 0x7408, 15961 47.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 23% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +4 8 0x7408, 13395 49.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +6 10 0x7408, 62036 44.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:22:23,227] INFO : Step 90000/100000 rank0 | Train Loss: 10.5888 | Train Loss Unweighted: 3.6256 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9282 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2656 | Regression_eta: 0.1044 | Regression_pt: 0.2178 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6354 | physical_nopu: 4.4354 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3451 | Classification_binary: 2.9100 | Regression_cos_phi: 1.8008 | Regression_energy: 1.3188 | Regression_eta: 1.1408 | Regression_pt: 1.5828 | Regression_sin_phi: 1.6100 | LR: 4.88e-05 | DataLoad Time: 0.0021s | Model Forward Time: 4.8165s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0033 | max_forward: 0.1953 | max_loss: 0.0645 | max_backward: 0.8258 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.9975 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5307.5 | uss_MB: 5074.7 | child_rss_MB: 7021.2 | allocated_MB: 404.6 | reserved_MB: 13422.0 | inactive_split_MB: 208.4 +rank0 [2026-10-27 21:22:23,491] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 2% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 9% +2 6 0x7408, 15961 47.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 23% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +4 8 0x7408, 13395 48.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 98% +6 10 0x7408, 62036 45.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:22:23,587] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/checkpoints/checkpoint-90000.pth +rank0 [2026-10-27 21:22:24,386] INFO : Running validation on rank0 +rank0 [2026-10-27 21:22:24,386] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 21:22:24,386] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 21:22:24,386] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 21:22:24,386] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 21:22:28,479] INFO : saved batch0_epoch90000.parquet +rank0 [2026-10-27 21:22:28,631] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 21:22:28,778] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 21:22:28,860] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 21:22:28,938] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 21:22:29,015] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 21:22:29,092] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 21:22:29,170] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 21:22:29,299] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 21:22:29,433] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 21:22:29,572] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 21:22:29,649] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 21:22:29,726] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 21:22:29,802] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 21:22:29,880] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 21:22:29,957] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 21:22:30,082] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 21:22:30,204] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 21:22:30,331] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 21:22:30,409] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 21:22:30,491] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 21:22:30,568] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 21:22:30,648] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 21:22:30,727] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 21:22:30,839] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 21:22:30,960] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 21:22:31,086] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 21:22:31,163] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 21:22:31,238] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 21:22:31,312] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 21:22:31,387] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 21:22:31,462] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 21:22:31,584] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 21:22:31,716] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 21:22:31,854] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 21:22:31,934] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 21:22:32,012] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 21:22:32,088] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 21:22:32,164] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 21:22:32,241] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 21:22:32,822] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 21:22:32,954] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 21:22:33,085] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 21:22:33,163] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 21:22:33,240] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 21:22:33,315] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 21:22:33,392] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 21:22:33,468] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 21:22:33,597] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 21:22:33,706] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 21:22:33,810] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 21:22:33,884] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 21:22:33,956] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 21:22:34,029] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 21:22:34,103] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 21:22:34,179] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 21:22:34,300] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 21:22:34,418] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 21:22:34,531] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 21:22:34,607] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 21:22:34,680] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 21:22:34,754] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 21:22:34,829] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 21:22:34,904] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 21:22:35,008] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 21:22:35,014] INFO : plotted pt_target histogram +rank0 [2026-10-27 21:22:35,017] INFO : plotted pt_pred histogram +rank0 [2026-10-27 21:22:35,022] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 21:22:35,024] INFO : plotted eta_target histogram +rank0 [2026-10-27 21:22:35,027] INFO : plotted eta_pred histogram +rank0 [2026-10-27 21:22:35,030] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 21:22:35,033] INFO : plotted sphi_target histogram +rank0 [2026-10-27 21:22:35,036] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 21:22:35,038] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 21:22:35,041] INFO : plotted cphi_target histogram +rank0 [2026-10-27 21:22:35,042] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 21:22:35,045] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 21:22:35,048] INFO : plotted energy_target histogram +rank0 [2026-10-27 21:22:35,051] INFO : plotted energy_pred histogram +rank0 [2026-10-27 21:22:35,054] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 21:24:24,226] INFO : VALIDATION | Step=90000/100000 | Train Loss=10.5888 | Valid Loss=11.6676 | Train Loss Unweighted=3.6256 | Valid Loss Unweighted=4.0383 | Group Valid Loss Unweighted: physical_pu: 3.6331 | physical_nopu: 4.4435 | Stale=0 | ETA=34.8m +rank0 [2026-10-27 21:24:24,235] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 21:24:24,387] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 21:24:24,390] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 21:24:48,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 21:24:49,515] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 21:24:49,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 21:24:50,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 21:24:51,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 21:24:51,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 21:24:52,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 21:24:52,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 21:25:02,466] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 21:25:02,472] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 21:25:02,869] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 21:25:02,873] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 21:25:18,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 21:25:18,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 21:25:19,521] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 21:25:19,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 21:25:20,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 21:25:20,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 21:25:21,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 21:25:21,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 21:25:31,911] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 21:25:32,094] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 21:25:32,491] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 21:25:32,495] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 21:25:47,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 21:25:48,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 21:25:48,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 21:25:49,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 21:25:49,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 21:25:50,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 21:25:50,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 21:25:51,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 21:26:01,231] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 21:26:01,285] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar +rank0 [2026-10-27 21:26:06,460] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar +rank0 [2026-10-27 21:26:07,619] INFO : Plotted number of elements +rank0 [2026-10-27 21:26:09,691] INFO : Plotted jets +rank0 [2026-10-27 21:26:10,877] INFO : Plotted jet ratio +rank0 [2026-10-27 21:26:10,919] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd +rank0 [2026-10-27 21:26:16,147] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd +rank0 [2026-10-27 21:26:17,374] INFO : Plotted number of elements +rank0 [2026-10-27 21:26:18,984] INFO : Plotted jets +rank0 [2026-10-27 21:26:20,084] INFO : Plotted jet ratio +rank0 [2026-10-27 21:26:20,122] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt +rank0 [2026-10-27 21:26:25,278] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt +rank0 [2026-10-27 21:26:26,510] INFO : Plotted number of elements +rank0 [2026-10-27 21:26:28,650] INFO : Plotted jets +rank0 [2026-10-27 21:26:29,820] INFO : Plotted jet ratio +rank0 [2026-10-27 21:29:49,033] INFO : Step 91000/100000 rank0 | Train Loss: 11.9808 | Train Loss Unweighted: 4.3520 | Train Loss Components: Classification: 0.0128 | Classification_binary: 3.5260 | Regression_cos_phi: 0.0522 | Regression_energy: 0.3480 | Regression_eta: 0.0723 | Regression_pt: 0.2901 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4346 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2481 | Classification_binary: 3.6463 | Regression_cos_phi: 1.9779 | Regression_energy: 1.5410 | Regression_eta: 0.7019 | Regression_pt: 1.9603 | Regression_sin_phi: 1.4905 | LR: 4.71e-05 | DataLoad Time: 0.0016s | Model Forward Time: 4.2977s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1743 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.3267 | max_tensorboard_logging: 2.0733 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0716 | Memory: rss_MB: 7308.2 | uss_MB: 7066.1 | child_rss_MB: 7044.9 | allocated_MB: 393.7 | reserved_MB: 11392.0 | inactive_split_MB: 89.1 +rank0 [2026-10-27 21:29:49,290] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 18% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 46.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +4 8 0x7408, 13395 47.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +6 10 0x7408, 62036 44.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:33:09,514] INFO : Step 92000/100000 rank0 | Train Loss: 14.4974 | Train Loss Unweighted: 4.7629 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4460 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5917 | Regression_eta: 0.0739 | Regression_pt: 0.5467 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6297 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3152 | Classification_binary: 3.5592 | Regression_cos_phi: 1.5994 | Regression_energy: 2.5592 | Regression_eta: 0.7529 | Regression_pt: 3.2865 | Regression_sin_phi: 1.5078 | LR: 4.57e-05 | DataLoad Time: 0.0021s | Model Forward Time: 3.4745s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1620 | max_loss: 0.0037 | max_backward: 0.2478 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 1.1676 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0595 | Memory: rss_MB: 7302.2 | uss_MB: 6481.6 | child_rss_MB: 6997.3 | allocated_MB: 394.0 | reserved_MB: 11392.0 | inactive_split_MB: 90.6 +rank0 [2026-10-27 21:33:09,772] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 19% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +2 6 0x7408, 15961 47.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +4 8 0x7408, 13395 49.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 98% +6 10 0x7408, 62036 45.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:36:11,954] INFO : Step 93000/100000 rank0 | Train Loss: 13.1041 | Train Loss Unweighted: 4.5189 | Train Loss Components: Classification: 0.0119 | Classification_binary: 3.5265 | Regression_cos_phi: 0.0575 | Regression_energy: 0.4342 | Regression_eta: 0.0649 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6346 | physical_nopu: 4.4327 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3004 | Classification_binary: 3.5188 | Regression_cos_phi: 1.4699 | Regression_energy: 2.3723 | Regression_eta: 0.8214 | Regression_pt: 3.1187 | Regression_sin_phi: 1.7185 | LR: 4.43e-05 | DataLoad Time: 0.0021s | Model Forward Time: 3.0779s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1013 | max_loss: 0.1069 | max_backward: 0.2035 | max_optimizer_step: 0.1792 | max_tensorboard_logging: 1.1596 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 5379.1 | uss_MB: 5141.2 | child_rss_MB: 6951.6 | allocated_MB: 393.6 | reserved_MB: 13294.0 | inactive_split_MB: 89.2 +rank0 [2026-10-27 21:36:12,213] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 20% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +2 6 0x7408, 15961 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +4 8 0x7408, 13395 49.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +6 10 0x7408, 62036 46.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:39:11,997] INFO : Step 94000/100000 rank0 | Train Loss: 11.5177 | Train Loss Unweighted: 4.2756 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4441 | Regression_cos_phi: 0.0413 | Regression_energy: 0.3503 | Regression_eta: 0.0686 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6308 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3423 | Classification_binary: 3.3369 | Regression_cos_phi: 1.4432 | Regression_energy: 1.4900 | Regression_eta: 0.6451 | Regression_pt: 1.9354 | Regression_sin_phi: 1.4785 | LR: 4.32e-05 | DataLoad Time: 0.0019s | Model Forward Time: 7.0081s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0191 | max_forward: 0.1721 | max_loss: 0.0188 | max_backward: 0.2028 | max_optimizer_step: 0.2895 | max_tensorboard_logging: 4.7123 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 5372.1 | uss_MB: 5157.2 | child_rss_MB: 6925.7 | allocated_MB: 394.1 | reserved_MB: 13294.0 | inactive_split_MB: 89.1 +rank0 [2026-10-27 21:39:12,258] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 20% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 99% +2 6 0x7408, 15961 47.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 18% 99% +4 8 0x7408, 13395 49.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +6 10 0x7408, 62036 45.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:42:21,361] INFO : Step 95000/100000 rank0 | Train Loss: 14.1207 | Train Loss Unweighted: 4.6477 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3920 | Regression_cos_phi: 0.0441 | Regression_energy: 0.5776 | Regression_eta: 0.0602 | Regression_pt: 0.5087 | Regression_sin_phi: 0.0499 | Group Train Loss Unweighted (window mean): physical_pu: 3.6267 | physical_nopu: 4.4321 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3267 | Classification_binary: 3.4085 | Regression_cos_phi: 2.0054 | Regression_energy: 2.0324 | Regression_eta: 0.7576 | Regression_pt: 2.5765 | Regression_sin_phi: 1.8980 | LR: 4.22e-05 | DataLoad Time: 0.0015s | Model Forward Time: 3.1684s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.0941 | max_loss: 0.0036 | max_backward: 0.1734 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 1.2811 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5361.0 | uss_MB: 5157.8 | child_rss_MB: 6928.3 | allocated_MB: 396.0 | reserved_MB: 14294.0 | inactive_split_MB: 91.1 +rank0 [2026-10-27 21:42:21,617] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 20% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 24% 100% +2 6 0x7408, 15961 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 18% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 49.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +6 10 0x7408, 62036 45.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:45:16,906] INFO : Step 96000/100000 rank0 | Train Loss: 11.5036 | Train Loss Unweighted: 4.2276 | Train Loss Components: Classification: 0.0097 | Classification_binary: 3.4897 | Regression_cos_phi: 0.0538 | Regression_energy: 0.2982 | Regression_eta: 0.0802 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6278 | physical_nopu: 4.4329 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2094 | Classification_binary: 3.4499 | Regression_cos_phi: 1.9207 | Regression_energy: 1.1765 | Regression_eta: 0.8730 | Regression_pt: 1.3541 | Regression_sin_phi: 1.5328 | LR: 4.14e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.0398s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0007 | max_forward: 0.0991 | max_loss: 0.0037 | max_backward: 0.1000 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5357.0 | uss_MB: 5158.4 | child_rss_MB: 6899.7 | allocated_MB: 393.0 | reserved_MB: 14294.0 | inactive_split_MB: 95.3 +rank0 [2026-10-27 21:45:17,172] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 21% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 24% 99% +2 6 0x7408, 15961 47.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 49.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +6 10 0x7408, 62036 45.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:48:35,261] INFO : Step 97000/100000 rank0 | Train Loss: 11.8793 | Train Loss Unweighted: 4.3044 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.4420 | Regression_cos_phi: 0.0454 | Regression_energy: 0.3663 | Regression_eta: 0.0711 | Regression_pt: 0.3156 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.6277 | physical_nopu: 4.4255 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3484 | Classification_binary: 3.5686 | Regression_cos_phi: 1.6453 | Regression_energy: 1.4179 | Regression_eta: 0.7734 | Regression_pt: 1.8839 | Regression_sin_phi: 1.7506 | LR: 4.08e-05 | DataLoad Time: 0.0013s | Model Forward Time: 3.2771s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1192 | max_loss: 0.0048 | max_backward: 0.1852 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 5356.0 | uss_MB: 5157.9 | child_rss_MB: 6903.2 | allocated_MB: 394.7 | reserved_MB: 14294.0 | inactive_split_MB: 98.1 +rank0 [2026-10-27 21:48:35,517] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 21% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 24% 100% +2 6 0x7408, 15961 47.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +6 10 0x7408, 62036 45.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:51:33,485] INFO : Step 98000/100000 rank0 | Train Loss: 12.9367 | Train Loss Unweighted: 4.4840 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0441 | Regression_energy: 0.4811 | Regression_eta: 0.0611 | Regression_pt: 0.4378 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3326 | Classification_binary: 3.3712 | Regression_cos_phi: 1.4365 | Regression_energy: 2.6298 | Regression_eta: 0.6214 | Regression_pt: 3.6874 | Regression_sin_phi: 1.3381 | LR: 4.04e-05 | DataLoad Time: 0.0015s | Model Forward Time: 3.5259s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1318 | max_loss: 0.0038 | max_backward: 0.1127 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.6868 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 5362.0 | uss_MB: 5160.3 | child_rss_MB: 6899.7 | allocated_MB: 394.5 | reserved_MB: 14294.0 | inactive_split_MB: 82.0 +rank0 [2026-10-27 21:51:33,748] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 21% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 24% 100% +2 6 0x7408, 15961 47.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 19% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 49.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +6 10 0x7408, 62036 44.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:54:36,495] INFO : Step 99000/100000 rank0 | Train Loss: 11.6408 | Train Loss Unweighted: 4.3204 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.5557 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3117 | Regression_eta: 0.0641 | Regression_pt: 0.2727 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.6292 | physical_nopu: 4.4284 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3224 | Classification_binary: 3.5327 | Regression_cos_phi: 1.7826 | Regression_energy: 1.4012 | Regression_eta: 0.6673 | Regression_pt: 1.8202 | Regression_sin_phi: 1.5425 | LR: 4.01e-05 | DataLoad Time: 0.0063s | Model Forward Time: 3.8353s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1731 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.8331 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 5358.0 | uss_MB: 5160.5 | child_rss_MB: 6886.0 | allocated_MB: 394.3 | reserved_MB: 14294.0 | inactive_split_MB: 113.7 +rank0 [2026-10-27 21:54:36,765] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 21% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 24% 100% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 100% +6 10 0x7408, 62036 44.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:57:57,193] INFO : Step 100000/100000 rank0 | Train Loss: 10.8909 | Train Loss Unweighted: 3.6545 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9133 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2833 | Regression_eta: 0.1041 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6336 | physical_nopu: 4.4311 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3503 | Classification_binary: 2.9187 | Regression_cos_phi: 1.6896 | Regression_energy: 1.3124 | Regression_eta: 1.1247 | Regression_pt: 1.6424 | Regression_sin_phi: 1.6163 | LR: 4.00e-05 | DataLoad Time: 0.0023s | Model Forward Time: 4.4274s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0030 | max_forward: 0.2088 | max_loss: 0.0393 | max_backward: 0.3031 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.9276 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 5350.9 | uss_MB: 5163.3 | child_rss_MB: 6834.2 | allocated_MB: 405.7 | reserved_MB: 14294.0 | inactive_split_MB: 165.9 +rank0 [2026-10-27 21:57:57,454] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 21% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 24% 99% +2 6 0x7408, 15961 47.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +4 8 0x7408, 13395 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 20% 29% +6 10 0x7408, 62036 44.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 20% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 19% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:57:57,663] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/checkpoints/checkpoint-100000.pth +rank0 [2026-10-27 21:57:58,522] INFO : Running validation on rank0 +rank0 [2026-10-27 21:57:58,522] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 21:57:58,522] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 21:57:58,522] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 21:57:58,522] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 21:58:01,964] INFO : saved batch0_epoch100000.parquet +rank0 [2026-10-27 21:58:02,136] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 21:58:02,280] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 21:58:02,359] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 21:58:02,433] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 21:58:02,507] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 21:58:02,581] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 21:58:02,660] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 21:58:02,791] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 21:58:02,921] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 21:58:03,052] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 21:58:03,128] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 21:58:03,205] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 21:58:03,286] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 21:58:03,364] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 21:58:03,441] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 21:58:03,574] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 21:58:03,695] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 21:58:03,816] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 21:58:03,894] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 21:58:03,969] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 21:58:04,043] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 21:58:04,120] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 21:58:04,195] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 21:58:04,308] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 21:58:04,429] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 21:58:04,549] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 21:58:04,626] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 21:58:04,702] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 21:58:04,777] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 21:58:04,851] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 21:58:04,924] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 21:58:05,042] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 21:58:05,181] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 21:58:05,311] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 21:58:05,390] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 21:58:05,465] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 21:58:05,540] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 21:58:05,615] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 21:58:05,691] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 21:58:06,285] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 21:58:06,414] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 21:58:06,541] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 21:58:06,618] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 21:58:06,691] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 21:58:06,763] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 21:58:06,838] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 21:58:06,912] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 21:58:07,041] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 21:58:07,154] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 21:58:07,262] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 21:58:07,335] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 21:58:07,406] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 21:58:07,477] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 21:58:07,548] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 21:58:07,622] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 21:58:07,740] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 21:58:07,855] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 21:58:07,967] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 21:58:08,041] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 21:58:08,114] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 21:58:08,187] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 21:58:08,262] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 21:58:08,337] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 21:58:08,443] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 21:58:08,448] INFO : plotted pt_target histogram +rank0 [2026-10-27 21:58:08,451] INFO : plotted pt_pred histogram +rank0 [2026-10-27 21:58:08,456] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 21:58:08,458] INFO : plotted eta_target histogram +rank0 [2026-10-27 21:58:08,461] INFO : plotted eta_pred histogram +rank0 [2026-10-27 21:58:08,464] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 21:58:08,467] INFO : plotted sphi_target histogram +rank0 [2026-10-27 21:58:08,469] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 21:58:08,472] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 21:58:08,475] INFO : plotted cphi_target histogram +rank0 [2026-10-27 21:58:08,478] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 21:58:08,481] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 21:58:08,484] INFO : plotted energy_target histogram +rank0 [2026-10-27 21:58:08,486] INFO : plotted energy_pred histogram +rank0 [2026-10-27 21:58:08,490] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 21:59:52,090] INFO : VALIDATION | Step=100000/100000 | Train Loss=10.8909 | Valid Loss=11.6527 | Train Loss Unweighted=3.6545 | Valid Loss Unweighted=4.0335 | Group Valid Loss Unweighted: physical_pu: 3.6303 | physical_nopu: 4.4368 | Stale=0 | ETA=0.0m +rank0 [2026-10-27 21:59:52,099] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 21:59:52,412] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 21:59:52,416] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 22:00:18,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 22:00:19,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 22:00:19,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 22:00:20,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 22:00:20,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 22:00:21,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 22:00:21,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 22:00:22,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 22:00:32,054] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 22:00:32,061] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 22:00:32,465] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 22:00:32,468] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 22:00:47,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 22:00:47,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 22:00:48,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 22:00:48,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 22:00:49,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 22:00:49,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 22:00:50,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 22:00:50,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 22:01:00,977] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 22:01:01,522] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 22:01:01,928] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 22:01:01,932] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 22:01:17,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 22:01:17,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 22:01:18,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 22:01:18,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 22:01:19,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 22:01:19,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 22:01:20,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 22:01:20,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 22:01:30,666] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 22:01:30,717] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar +rank0 [2026-10-27 22:01:35,877] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar +rank0 [2026-10-27 22:01:37,036] INFO : Plotted number of elements +rank0 [2026-10-27 22:01:39,123] INFO : Plotted jets +rank0 [2026-10-27 22:01:40,310] INFO : Plotted jet ratio +rank0 [2026-10-27 22:01:40,376] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd +rank0 [2026-10-27 22:01:45,490] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd +rank0 [2026-10-27 22:01:46,690] INFO : Plotted number of elements +rank0 [2026-10-27 22:01:48,328] INFO : Plotted jets +rank0 [2026-10-27 22:01:49,426] INFO : Plotted jet ratio +rank0 [2026-10-27 22:01:49,464] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt +rank0 [2026-10-27 22:01:54,669] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt +rank0 [2026-10-27 22:01:55,934] INFO : Plotted number of elements +rank0 [2026-10-27 22:01:58,110] INFO : Plotted jets +rank0 [2026-10-27 22:01:59,334] INFO : Plotted jet ratio +rank0 [2026-10-27 22:01:59,729] INFO : Training completed. Total time on device 0: 350.713min diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.1 b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.1 new file mode 100644 index 0000000000000000000000000000000000000000..926e5e87573fd4580f9606573ab1fc43217cf14d --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.1 @@ -0,0 +1,851 @@ +rank1 [2026-10-27 16:10:10,462] INFO : Initializing model with seed=12345; process seed=12346 +rank1 [2026-10-27 16:10:10,463] INFO : configured dtype=torch.bfloat16 for autocast +rank1 [2026-10-27 16:10:37,169] INFO : Instantiating model +rank1 [2026-10-27 16:10:37,169] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank1 [2026-10-27 16:10:37,169] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank1 [2026-10-27 16:10:37,201] INFO : Input encoding initialization took 0.03s +rank1 [2026-10-27 16:10:37,201] INFO : nn0 parameters: 2130944 +rank1 [2026-10-27 16:10:37,201] INFO : Initializing shared backbone layers, num_convs=3 +rank1 [2026-10-27 16:10:37,201] INFO : Initializing backbone layer 0 +rank1 [2026-10-27 16:10:37,201] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-27 16:10:37,212] INFO : Initializing backbone layer 1 +rank1 [2026-10-27 16:10:37,212] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-27 16:10:37,223] INFO : Initializing backbone layer 2 +rank1 [2026-10-27 16:10:37,223] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-27 16:10:37,233] INFO : Backbone initialization took 0.03s +rank1 [2026-10-27 16:10:37,233] INFO : backbone parameters: 4733952 +rank1 [2026-10-27 16:10:37,233] INFO : Initializing output DNNs +rank1 [2026-10-27 16:10:37,267] INFO : Output DNNs initialization took 0.03s +rank1 [2026-10-27 16:10:37,268] INFO : backbone_mode=BackboneMode.SHARED +rank1 [2026-10-27 16:10:37,268] INFO : jagged_attention enabled=True +rank1 [2026-10-27 16:10:37,268] INFO : flash_attn_varlen enabled=True +rank1 [2026-10-27 16:10:37,268] INFO : task_queries enabled=False +rank1 [2026-10-27 16:10:37,268] INFO : classification_norm parameters: 1024 +rank1 [2026-10-27 16:10:37,268] INFO : regression_norm parameters: 1024 +rank1 [2026-10-27 16:10:37,268] INFO : classification_query parameters: 0 +rank1 [2026-10-27 16:10:37,268] INFO : regression_query parameters: 0 +rank1 [2026-10-27 16:10:37,268] INFO : classification_readout parameters: 0 +rank1 [2026-10-27 16:10:37,268] INFO : regression_readout parameters: 0 +rank1 [2026-10-27 16:10:37,268] INFO : output_mode=elementwise +rank1 [2026-10-27 16:10:37,268] INFO : set_decoder parameters: 0 +rank1 [2026-10-27 16:10:37,268] INFO : nn_binary_particle parameters: 264706 +rank1 [2026-10-27 16:10:37,268] INFO : nn_pid parameters: 268297 +rank1 [2026-10-27 16:10:37,268] INFO : nn_pt parameters: 2113544 +rank1 [2026-10-27 16:10:37,268] INFO : nn_eta parameters: 264706 +rank1 [2026-10-27 16:10:37,269] INFO : nn_sin_phi parameters: 264706 +rank1 [2026-10-27 16:10:37,269] INFO : nn_cos_phi parameters: 264706 +rank1 [2026-10-27 16:10:37,269] INFO : nn_energy parameters: 2113544 +rank1 [2026-10-27 16:10:37,269] INFO : Total MLPF parameters: 12421153 +rank1 [2026-10-27 16:10:37,269] INFO : MLPF __init__ done +rank1 [2026-10-27 16:10:37,270] INFO : Instantiated model +rank1 [2026-10-27 16:10:37,270] INFO : Moving model to device rank=1 +rank1 [2026-10-27 16:10:37,300] INFO : Moved model to device rank=1 +rank1 [2026-10-27 16:10:37,301] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank1 [2026-10-27 16:10:37,302] INFO : Configured model for SyncBatchNorm rank=1 +rank1 [2026-10-27 16:10:37,493] INFO : Configured model for DistributedDataParallel rank=1 +rank1 [2026-10-27 16:10:37,594] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +167 module.nn_energy.nn.7.0.bias ... 0 +168 module.nn_energy.nn.7.2.weight ... 0 +169 module.nn_energy.nn.7.2.bias ... 0 +170 module.nn_energy.nn.7.4.weight ... 0 +171 module.nn_energy.nn.7.4.bias ... 0 + +[172 rows x 3 columns] +rank1 [2026-10-27 16:10:37,594] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=512, out_features=512, bias=True) + ) + (norm0): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=512, out_features=512, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank1 [2026-10-27 16:10:37,595] INFO : Trainable parameters: 12421153 +rank1 [2026-10-27 16:10:37,595] INFO : Non-trainable parameters: 0 +rank1 [2026-10-27 16:10:37,595] INFO : Total parameters: 12421153 +rank1 [2026-10-27 16:10:37,597] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.mha.in_proj_weight 786432 0 + module.backbone.0.mha.in_proj_bias 1536 0 +module.backbone.0.mha.out_proj.weight 262144 0 + module.backbone.0.mha.out_proj.bias 512 0 + module.backbone.0.norm0.weight 512 0 + module.backbone.0.norm0.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm1.bias 512 0 + module.backbone.0.seq.0.weight 262144 0 + module.backbone.0.seq.0.bias 512 0 + module.backbone.0.seq.2.weight 262144 0 + module.backbone.0.seq.2.bias 512 0 + module.backbone.1.mha.in_proj_weight 786432 0 + module.backbone.1.mha.in_proj_bias 1536 0 +module.backbone.1.mha.out_proj.weight 262144 0 + module.backbone.1.mha.out_proj.bias 512 0 + module.backbone.1.norm0.weight 512 0 + module.backbone.1.norm0.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm1.bias 512 0 + module.backbone.1.seq.0.weight 262144 0 + module.backbone.1.seq.0.bias 512 0 + module.backbone.1.seq.2.weight 262144 0 + module.backbone.1.seq.2.bias 512 0 + module.backbone.2.mha.in_proj_weight 786432 0 + module.backbone.2.mha.in_proj_bias 1536 0 +module.backbone.2.mha.out_proj.weight 262144 0 + module.backbone.2.mha.out_proj.bias 512 0 + module.backbone.2.norm0.weight 512 0 + module.backbone.2.norm0.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm1.bias 512 0 + module.backbone.2.seq.0.weight 262144 0 + module.backbone.2.seq.0.bias 512 0 + module.backbone.2.seq.2.weight 262144 0 + module.backbone.2.seq.2.bias 512 0 + module.classification_norm.weight 512 0 + module.classification_norm.bias 512 0 + module.regression_norm.weight 512 0 + module.regression_norm.bias 512 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank1 [2026-10-27 16:10:37,597] INFO : Getting dataloaders +rank1 [2026-10-27 16:10:37,597] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:43,313] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:44,041] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:44,830] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank1 [2026-10-27 16:10:44,830] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:45,823] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:46,991] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:47,782] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank1 [2026-10-27 16:10:47,782] INFO : Creating WeightedMixIterator with 2 data loaders. +rank1 [2026-10-27 16:10:47,782] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:48,203] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:48,361] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:48,508] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank1 [2026-10-27 16:10:48,508] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:48,677] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:48,838] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:48,979] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank1 [2026-10-27 16:10:48,979] INFO : Creating InterleavedIterator with 2 data loaders. +rank1 [2026-10-27 16:11:16,549] INFO : Got dataloaders +rank1 [2026-10-27 16:11:16,550] INFO : loader split=train rank=1 len=232058 +rank1 [2026-10-27 16:11:16,550] INFO : loader split=valid rank=1 len=48 +rank1 [2026-10-27 16:11:16,550] INFO : Starting training from step 1 to 100000 on rank 1 of 8 +rank1 [2026-10-27 16:11:16,550] INFO : Creating train_iterator +rank1 [2026-10-27 16:11:16,550] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank1 [2026-10-27 16:11:30,493] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank1 [2026-10-27 16:11:45,738] INFO : Created train_iterator +rank1 [2026-10-27 16:12:23,641] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.123548031449318, 'Classification': 0.2298217851272784, 'Regression_pt': 0.7194055180624127, 'Regression_eta': 0.4619348940253258, 'Regression_sin_phi': 0.15005176125559955, 'Regression_cos_phi': 0.14165307599119842, 'Regression_energy': 1.0569261094555258}, weights={'Classification_binary': 1.0, 'Classification': 22.293569564819336, 'Regression_pt': 7.12191915512085, 'Regression_eta': 11.091493606567383, 'Regression_sin_phi': 34.14520263671875, 'Regression_cos_phi': 36.16969299316406, 'Regression_energy': 4.847593307495117} +rank1 [2026-10-27 16:14:48,546] INFO : Step 1000/100000 rank1 | Train Loss: 15.9738 | Train Loss Unweighted: 5.5021 | Train Loss Components: Classification: 0.0191 | Classification_binary: 4.1436 | Regression_cos_phi: 0.0516 | Regression_energy: 0.6035 | Regression_eta: 0.0819 | Regression_pt: 0.5500 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 5.2846 | physical_nopu: 6.1663 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4346 | Classification_binary: 4.1157 | Regression_cos_phi: 1.6889 | Regression_energy: 2.9572 | Regression_eta: 0.9206 | Regression_pt: 3.9764 | Regression_sin_phi: 2.0411 | LR: 4.00e-04 | DataLoad Time: 0.0008s | Model Forward Time: 1.9591s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0013 | max_forward: 0.0194 | max_loss: 0.0039 | max_backward: 0.0308 | max_optimizer_step: 0.1391 | max_tensorboard_logging: 1.0816 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0030 | Memory: rss_MB: 4094.8 | uss_MB: 2885.6 | child_rss_MB: 10815.8 | allocated_MB: 395.5 | reserved_MB: 11198.0 | inactive_split_MB: 104.9 +rank1 [2026-10-27 16:17:31,218] INFO : Step 2000/100000 rank1 | Train Loss: 17.2752 | Train Loss Unweighted: 5.6831 | Train Loss Components: Classification: 0.0157 | Classification_binary: 4.0887 | Regression_cos_phi: 0.0506 | Regression_energy: 0.7236 | Regression_eta: 0.0696 | Regression_pt: 0.6798 | Regression_sin_phi: 0.0552 | Group Train Loss Unweighted (window mean): physical_pu: 4.4021 | physical_nopu: 5.4797 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3830 | Classification_binary: 4.1389 | Regression_cos_phi: 1.7676 | Regression_energy: 3.9214 | Regression_eta: 0.7578 | Regression_pt: 5.5211 | Regression_sin_phi: 1.5142 | LR: 4.00e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.7794s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0250 | max_loss: 0.0036 | max_backward: 0.0657 | max_optimizer_step: 0.1397 | max_tensorboard_logging: 0.8949 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4097.7 | uss_MB: 2895.6 | child_rss_MB: 10834.7 | allocated_MB: 394.2 | reserved_MB: 11198.0 | inactive_split_MB: 90.1 +rank1 [2026-10-27 16:20:19,985] INFO : Step 3000/100000 rank1 | Train Loss: 14.0295 | Train Loss Unweighted: 5.1258 | Train Loss Components: Classification: 0.0148 | Classification_binary: 4.0405 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4749 | Regression_eta: 0.0746 | Regression_pt: 0.4205 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 4.1913 | physical_nopu: 5.3261 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3042 | Classification_binary: 3.8541 | Regression_cos_phi: 1.3576 | Regression_energy: 2.5270 | Regression_eta: 0.6477 | Regression_pt: 3.5281 | Regression_sin_phi: 1.5096 | LR: 3.99e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.4933s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1067 | max_loss: 0.0036 | max_backward: 0.0365 | max_optimizer_step: 0.1415 | max_tensorboard_logging: 1.1098 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0093 | Memory: rss_MB: 4102.6 | uss_MB: 2907.4 | child_rss_MB: 10837.4 | allocated_MB: 395.4 | reserved_MB: 11784.0 | inactive_split_MB: 88.5 +rank1 [2026-10-27 16:23:08,050] INFO : Step 4000/100000 rank1 | Train Loss: 11.7244 | Train Loss Unweighted: 3.9925 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.1718 | Regression_cos_phi: 0.0487 | Regression_energy: 0.3212 | Regression_eta: 0.1125 | Regression_pt: 0.2729 | Regression_sin_phi: 0.0492 | Group Train Loss Unweighted (window mean): physical_pu: 4.0838 | physical_nopu: 5.2370 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3512 | Classification_binary: 3.1753 | Regression_cos_phi: 1.7862 | Regression_energy: 1.6529 | Regression_eta: 1.2232 | Regression_pt: 2.0690 | Regression_sin_phi: 1.7307 | LR: 3.99e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.2687s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0037 | max_forward: 0.0726 | max_loss: 0.0066 | max_backward: 0.1670 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4108.6 | uss_MB: 2925.8 | child_rss_MB: 10838.2 | allocated_MB: 406.4 | reserved_MB: 11904.0 | inactive_split_MB: 179.6 +rank1 [2026-10-27 16:25:55,983] INFO : Step 5000/100000 rank1 | Train Loss: 13.1313 | Train Loss Unweighted: 4.9481 | Train Loss Components: Classification: 0.0138 | Classification_binary: 4.0144 | Regression_cos_phi: 0.0518 | Regression_energy: 0.3902 | Regression_eta: 0.0928 | Regression_pt: 0.3380 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 4.0057 | physical_nopu: 5.1582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2917 | Classification_binary: 3.9276 | Regression_cos_phi: 1.6797 | Regression_energy: 1.7704 | Regression_eta: 0.9083 | Regression_pt: 2.2514 | Regression_sin_phi: 1.9329 | LR: 3.98e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.3571s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0196 | max_loss: 0.0037 | max_backward: 0.0324 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 1.3896 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0068 | Memory: rss_MB: 4091.1 | uss_MB: 2927.1 | child_rss_MB: 10673.2 | allocated_MB: 393.9 | reserved_MB: 11904.0 | inactive_split_MB: 90.0 +rank1 [2026-10-27 16:28:58,361] INFO : Step 6000/100000 rank1 | Train Loss: 14.5226 | Train Loss Unweighted: 5.0639 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.8601 | Regression_cos_phi: 0.0525 | Regression_energy: 0.5359 | Regression_eta: 0.0694 | Regression_pt: 0.4825 | Regression_sin_phi: 0.0459 | Group Train Loss Unweighted (window mean): physical_pu: 3.9666 | physical_nopu: 5.0829 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3535 | Classification_binary: 4.0517 | Regression_cos_phi: 2.0016 | Regression_energy: 2.1499 | Regression_eta: 0.7436 | Regression_pt: 2.7949 | Regression_sin_phi: 1.4735 | LR: 3.97e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.5986s | Phase Times: max_pre_step_barrier: 0.0280 | max_batch_to_device: 0.0008 | max_forward: 0.0311 | max_loss: 0.0037 | max_backward: 0.0629 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 2.6604 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0119 | Memory: rss_MB: 4078.1 | uss_MB: 2934.4 | child_rss_MB: 10495.3 | allocated_MB: 394.6 | reserved_MB: 12524.0 | inactive_split_MB: 109.0 +rank1 [2026-10-27 16:31:51,870] INFO : Step 7000/100000 rank1 | Train Loss: 13.9772 | Train Loss Unweighted: 5.0398 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.9636 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4671 | Regression_eta: 0.0797 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0537 | Group Train Loss Unweighted (window mean): physical_pu: 3.9390 | physical_nopu: 5.0316 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3391 | Classification_binary: 3.9328 | Regression_cos_phi: 1.7173 | Regression_energy: 2.6393 | Regression_eta: 0.9372 | Regression_pt: 3.3519 | Regression_sin_phi: 1.8980 | LR: 3.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.1749s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0243 | max_loss: 0.0036 | max_backward: 0.0295 | max_optimizer_step: 0.2429 | max_tensorboard_logging: 2.1406 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 4070.6 | uss_MB: 2937.8 | child_rss_MB: 10370.1 | allocated_MB: 393.9 | reserved_MB: 12524.0 | inactive_split_MB: 107.5 +rank1 [2026-10-27 16:35:03,086] INFO : Step 8000/100000 rank1 | Train Loss: 13.6161 | Train Loss Unweighted: 4.9650 | Train Loss Components: Classification: 0.0187 | Classification_binary: 3.9275 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4519 | Regression_eta: 0.0615 | Regression_pt: 0.4056 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.9031 | physical_nopu: 4.9878 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4545 | Classification_binary: 3.9402 | Regression_cos_phi: 1.8365 | Regression_energy: 2.5440 | Regression_eta: 0.7578 | Regression_pt: 3.1168 | Regression_sin_phi: 1.5109 | LR: 3.94e-04 | DataLoad Time: 0.0014s | Model Forward Time: 2.9578s | Phase Times: max_pre_step_barrier: 0.0692 | max_batch_to_device: 0.0893 | max_forward: 0.1169 | max_loss: 0.0037 | max_backward: 0.0488 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 1.9012 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4069.6 | uss_MB: 2945.4 | child_rss_MB: 10376.1 | allocated_MB: 393.8 | reserved_MB: 12524.0 | inactive_split_MB: 105.1 +rank1 [2026-10-27 16:38:07,476] INFO : Step 9000/100000 rank1 | Train Loss: 15.1606 | Train Loss Unweighted: 5.0855 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.7493 | Regression_cos_phi: 0.0474 | Regression_energy: 0.5909 | Regression_eta: 0.0749 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.8927 | physical_nopu: 4.9579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3784 | Classification_binary: 3.6862 | Regression_cos_phi: 2.1256 | Regression_energy: 2.9262 | Regression_eta: 0.8741 | Regression_pt: 4.2749 | Regression_sin_phi: 1.2532 | LR: 3.93e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.8502s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0007 | max_forward: 0.0386 | max_loss: 0.0041 | max_backward: 0.0582 | max_optimizer_step: 0.1496 | max_tensorboard_logging: 1.9225 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 4056.6 | uss_MB: 2948.9 | child_rss_MB: 10378.8 | allocated_MB: 395.8 | reserved_MB: 12524.0 | inactive_split_MB: 89.5 +rank1 [2026-10-27 16:41:18,312] INFO : Step 10000/100000 rank1 | Train Loss: 12.2525 | Train Loss Unweighted: 4.6725 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8667 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3374 | Regression_eta: 0.0750 | Regression_pt: 0.2772 | Regression_sin_phi: 0.0525 | Group Train Loss Unweighted (window mean): physical_pu: 3.8762 | physical_nopu: 4.9262 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3267 | Classification_binary: 3.9045 | Regression_cos_phi: 1.7710 | Regression_energy: 1.7265 | Regression_eta: 0.9405 | Regression_pt: 1.9622 | Regression_sin_phi: 1.9804 | LR: 3.91e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.1911s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0277 | max_loss: 0.0035 | max_backward: 0.0304 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 2.2107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4024.3 | uss_MB: 2953.8 | child_rss_MB: 10382.4 | allocated_MB: 393.6 | reserved_MB: 12524.0 | inactive_split_MB: 89.8 +rank1 [2026-10-27 16:41:19,251] INFO : Running validation on rank1 +rank1 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 16:43:24,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:43:25,081] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 16:43:52,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 16:43:52,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 16:43:53,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 16:43:53,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 16:43:54,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 16:43:54,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 16:43:55,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 16:43:55,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 16:44:05,857] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-27 16:44:06,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:44:06,938] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 16:44:22,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 16:44:22,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 16:44:23,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 16:44:23,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 16:44:24,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 16:44:24,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 16:44:25,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 16:44:25,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 16:44:35,617] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 16:44:35,782] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:44:36,273] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 16:44:51,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 16:44:51,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 16:44:52,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 16:44:52,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 16:44:53,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 16:44:53,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 16:44:54,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 16:44:54,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 16:45:04,519] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 16:48:51,455] INFO : Step 11000/100000 rank1 | Train Loss: 13.5331 | Train Loss Unweighted: 4.8490 | Train Loss Components: Classification: 0.0178 | Classification_binary: 3.8332 | Regression_cos_phi: 0.0514 | Regression_energy: 0.4326 | Regression_eta: 0.0715 | Regression_pt: 0.3906 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.8629 | physical_nopu: 4.8860 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3486 | Classification_binary: 3.8413 | Regression_cos_phi: 2.4094 | Regression_energy: 2.1940 | Regression_eta: 0.8035 | Regression_pt: 2.7017 | Regression_sin_phi: 2.0579 | LR: 3.89e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.1304s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0010 | max_forward: 0.0938 | max_loss: 0.0036 | max_backward: 0.1077 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.9405 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6748.2 | uss_MB: 5693.6 | child_rss_MB: 10378.2 | allocated_MB: 394.1 | reserved_MB: 11016.0 | inactive_split_MB: 102.9 +rank1 [2026-10-27 16:51:46,972] INFO : Step 12000/100000 rank1 | Train Loss: 16.0880 | Train Loss Unweighted: 5.2955 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.7102 | Regression_cos_phi: 0.0409 | Regression_energy: 0.7300 | Regression_eta: 0.0621 | Regression_pt: 0.6963 | Regression_sin_phi: 0.0392 | Group Train Loss Unweighted (window mean): physical_pu: 3.8522 | physical_nopu: 4.8676 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3109 | Classification_binary: 3.8829 | Regression_cos_phi: 1.5115 | Regression_energy: 3.6736 | Regression_eta: 0.7751 | Regression_pt: 5.0318 | Regression_sin_phi: 1.7487 | LR: 3.87e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.9531s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1050 | max_forward: 0.2073 | max_loss: 0.0038 | max_backward: 0.1040 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 1.6163 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6734.2 | uss_MB: 5695.7 | child_rss_MB: 10317.1 | allocated_MB: 394.0 | reserved_MB: 11384.0 | inactive_split_MB: 88.0 +rank1 [2026-10-27 16:54:47,130] INFO : Step 13000/100000 rank1 | Train Loss: 12.2526 | Train Loss Unweighted: 4.6994 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.8775 | Regression_cos_phi: 0.0480 | Regression_energy: 0.3426 | Regression_eta: 0.0688 | Regression_pt: 0.2970 | Regression_sin_phi: 0.0539 | Group Train Loss Unweighted (window mean): physical_pu: 3.8404 | physical_nopu: 4.8415 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2492 | Classification_binary: 3.8152 | Regression_cos_phi: 1.8961 | Regression_energy: 2.2451 | Regression_eta: 1.1734 | Regression_pt: 2.8836 | Regression_sin_phi: 2.5795 | LR: 3.85e-04 | DataLoad Time: 0.0071s | Model Forward Time: 3.3125s | Phase Times: max_pre_step_barrier: 0.0143 | max_batch_to_device: 0.0010 | max_forward: 0.0740 | max_loss: 0.0036 | max_backward: 0.0639 | max_optimizer_step: 0.2642 | max_tensorboard_logging: 2.1016 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 5345.9 | uss_MB: 4318.0 | child_rss_MB: 10308.5 | allocated_MB: 394.7 | reserved_MB: 12168.0 | inactive_split_MB: 111.5 +rank1 [2026-10-27 16:57:46,633] INFO : Step 14000/100000 rank1 | Train Loss: 11.1328 | Train Loss Unweighted: 3.8032 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.0595 | Regression_cos_phi: 0.0494 | Regression_energy: 0.2823 | Regression_eta: 0.1077 | Regression_pt: 0.2396 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.8281 | physical_nopu: 4.8178 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3865 | Classification_binary: 3.0198 | Regression_cos_phi: 1.7967 | Regression_energy: 1.4112 | Regression_eta: 1.3570 | Regression_pt: 1.7213 | Regression_sin_phi: 1.6645 | LR: 3.83e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.7657s | Phase Times: max_pre_step_barrier: 0.1064 | max_batch_to_device: 0.0033 | max_forward: 0.1130 | max_loss: 0.0036 | max_backward: 0.2788 | max_optimizer_step: 0.1663 | max_tensorboard_logging: 2.1758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1099 | Memory: rss_MB: 5335.9 | uss_MB: 4329.8 | child_rss_MB: 10259.4 | allocated_MB: 405.3 | reserved_MB: 12168.0 | inactive_split_MB: 126.3 +rank1 [2026-10-27 17:00:50,804] INFO : Step 15000/100000 rank1 | Train Loss: 12.4223 | Train Loss Unweighted: 4.6301 | Train Loss Components: Classification: 0.0132 | Classification_binary: 3.7975 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3484 | Regression_eta: 0.0765 | Regression_pt: 0.2883 | Regression_sin_phi: 0.0502 | Group Train Loss Unweighted (window mean): physical_pu: 3.8205 | physical_nopu: 4.8025 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3633 | Classification_binary: 3.6002 | Regression_cos_phi: 1.5053 | Regression_energy: 2.2451 | Regression_eta: 0.7708 | Regression_pt: 2.5127 | Regression_sin_phi: 1.5991 | LR: 3.80e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.2226s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0799 | max_loss: 0.0851 | max_backward: 0.1383 | max_optimizer_step: 0.1621 | max_tensorboard_logging: 2.1079 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5320.9 | uss_MB: 4324.4 | child_rss_MB: 10137.7 | allocated_MB: 394.1 | reserved_MB: 12168.0 | inactive_split_MB: 89.6 +rank1 [2026-10-27 17:04:12,219] INFO : Step 16000/100000 rank1 | Train Loss: 13.1559 | Train Loss Unweighted: 4.6950 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.6917 | Regression_cos_phi: 0.0473 | Regression_energy: 0.4330 | Regression_eta: 0.0771 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.8147 | physical_nopu: 4.7795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3338 | Classification_binary: 3.8867 | Regression_cos_phi: 1.8457 | Regression_energy: 2.1160 | Regression_eta: 1.3279 | Regression_pt: 2.6078 | Regression_sin_phi: 1.6813 | LR: 3.78e-04 | DataLoad Time: 0.0032s | Model Forward Time: 3.2745s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.0598 | max_loss: 0.0036 | max_backward: 0.1881 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 2.1012 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5304.0 | uss_MB: 4326.3 | child_rss_MB: 10027.9 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 88.9 +rank1 [2026-10-27 17:07:14,836] INFO : Step 17000/100000 rank1 | Train Loss: 13.6189 | Train Loss Unweighted: 4.8180 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7348 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4756 | Regression_eta: 0.0731 | Regression_pt: 0.4225 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.8095 | physical_nopu: 4.7681 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3332 | Classification_binary: 3.6864 | Regression_cos_phi: 1.9342 | Regression_energy: 1.8597 | Regression_eta: 0.8597 | Regression_pt: 2.2715 | Regression_sin_phi: 2.0720 | LR: 3.75e-04 | DataLoad Time: 0.6514s | Model Forward Time: 3.3910s | Phase Times: max_pre_step_barrier: 0.6520 | max_batch_to_device: 0.0829 | max_forward: 0.1388 | max_loss: 0.0036 | max_backward: 0.0636 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 2.2240 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5273.0 | uss_MB: 4329.2 | child_rss_MB: 9776.7 | allocated_MB: 394.3 | reserved_MB: 12168.0 | inactive_split_MB: 88.4 +rank1 [2026-10-27 17:10:19,258] INFO : Step 18000/100000 rank1 | Train Loss: 12.6032 | Train Loss Unweighted: 4.6266 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7426 | Regression_cos_phi: 0.0507 | Regression_energy: 0.3730 | Regression_eta: 0.0736 | Regression_pt: 0.3195 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7954 | physical_nopu: 4.7518 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2663 | Classification_binary: 3.7183 | Regression_cos_phi: 1.6085 | Regression_energy: 1.4998 | Regression_eta: 0.6750 | Regression_pt: 2.0306 | Regression_sin_phi: 1.5880 | LR: 3.72e-04 | DataLoad Time: 0.0018s | Model Forward Time: 4.2069s | Phase Times: max_pre_step_barrier: 0.0217 | max_batch_to_device: 0.0009 | max_forward: 0.1212 | max_loss: 0.0040 | max_backward: 0.1670 | max_optimizer_step: 0.1610 | max_tensorboard_logging: 3.0399 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5245.9 | uss_MB: 4331.6 | child_rss_MB: 9623.0 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 87.1 +rank1 [2026-10-27 17:13:21,531] INFO : Step 19000/100000 rank1 | Train Loss: 14.6123 | Train Loss Unweighted: 5.0036 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.6735 | Regression_cos_phi: 0.0398 | Regression_energy: 0.6019 | Regression_eta: 0.0612 | Regression_pt: 0.5669 | Regression_sin_phi: 0.0441 | Group Train Loss Unweighted (window mean): physical_pu: 3.7887 | physical_nopu: 4.7426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4372 | Classification_binary: 3.5978 | Regression_cos_phi: 1.5729 | Regression_energy: 4.9772 | Regression_eta: 0.7113 | Regression_pt: 7.1892 | Regression_sin_phi: 1.4917 | LR: 3.69e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.4967s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1593 | max_loss: 0.0094 | max_backward: 0.0749 | max_optimizer_step: 0.1658 | max_tensorboard_logging: 2.3024 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5218.9 | uss_MB: 4333.7 | child_rss_MB: 9466.4 | allocated_MB: 394.5 | reserved_MB: 12168.0 | inactive_split_MB: 103.1 +rank1 [2026-10-27 17:16:34,024] INFO : Step 20000/100000 rank1 | Train Loss: 13.1046 | Train Loss Unweighted: 4.7571 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.7799 | Regression_cos_phi: 0.0517 | Regression_energy: 0.4251 | Regression_eta: 0.0654 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0511 | Group Train Loss Unweighted (window mean): physical_pu: 3.7835 | physical_nopu: 4.7206 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2598 | Classification_binary: 3.8302 | Regression_cos_phi: 1.8117 | Regression_energy: 2.0037 | Regression_eta: 0.7593 | Regression_pt: 2.5314 | Regression_sin_phi: 1.5474 | LR: 3.66e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.2230s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0008 | max_forward: 0.1344 | max_loss: 0.0037 | max_backward: 0.0763 | max_optimizer_step: 0.1649 | max_tensorboard_logging: 1.9693 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5216.3 | uss_MB: 4336.2 | child_rss_MB: 9462.6 | allocated_MB: 392.8 | reserved_MB: 12168.0 | inactive_split_MB: 89.4 +rank1 [2026-10-27 17:16:34,962] INFO : Running validation on rank1 +rank1 [2026-10-27 17:16:34,962] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 17:16:34,962] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 17:16:34,962] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 17:16:34,962] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 17:18:27,486] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 17:18:27,879] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 17:18:53,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 17:18:54,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 17:18:54,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 17:18:55,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 17:18:55,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 17:18:56,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 17:18:56,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 17:18:57,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 17:19:07,010] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 17:19:07,312] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 17:19:07,733] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 17:19:22,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 17:19:23,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 17:19:23,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 17:19:24,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 17:19:24,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 17:19:25,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 17:19:25,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 17:19:26,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 17:19:36,191] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 17:19:36,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 17:19:36,812] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 17:19:51,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 17:19:52,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 17:19:52,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 17:19:53,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 17:19:53,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 17:19:53,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 17:19:54,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 17:19:54,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 17:20:04,812] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 17:23:25,419] INFO : Step 21000/100000 rank1 | Train Loss: 12.4320 | Train Loss Unweighted: 4.5235 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.6041 | Regression_cos_phi: 0.0444 | Regression_energy: 0.3902 | Regression_eta: 0.0812 | Regression_pt: 0.3383 | Regression_sin_phi: 0.0478 | Group Train Loss Unweighted (window mean): physical_pu: 3.7732 | physical_nopu: 4.7052 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3994 | Classification_binary: 3.7111 | Regression_cos_phi: 1.2805 | Regression_energy: 1.8257 | Regression_eta: 1.0418 | Regression_pt: 2.2998 | Regression_sin_phi: 1.4860 | LR: 3.62e-04 | DataLoad Time: 0.0026s | Model Forward Time: 3.7287s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.2105 | max_loss: 0.0056 | max_backward: 0.1347 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 2.3606 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5326.2 | uss_MB: 4454.3 | child_rss_MB: 9450.4 | allocated_MB: 394.6 | reserved_MB: 11062.0 | inactive_split_MB: 89.8 +rank1 [2026-10-27 17:26:14,701] INFO : Step 22000/100000 rank1 | Train Loss: 14.5232 | Train Loss Unweighted: 4.8893 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.6034 | Regression_cos_phi: 0.0476 | Regression_energy: 0.5714 | Regression_eta: 0.0738 | Regression_pt: 0.5351 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.7709 | physical_nopu: 4.6938 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3476 | Classification_binary: 3.6412 | Regression_cos_phi: 2.0400 | Regression_energy: 3.3147 | Regression_eta: 1.3021 | Regression_pt: 4.6382 | Regression_sin_phi: 1.3910 | LR: 3.59e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.6775s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0010 | max_forward: 0.0774 | max_loss: 0.0142 | max_backward: 0.0921 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.4442 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1027 | Memory: rss_MB: 5327.2 | uss_MB: 4458.5 | child_rss_MB: 9447.9 | allocated_MB: 394.2 | reserved_MB: 11488.0 | inactive_split_MB: 86.8 +rank1 [2026-10-27 17:29:50,014] INFO : Step 23000/100000 rank1 | Train Loss: 12.2140 | Train Loss Unweighted: 4.5907 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.7888 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3388 | Regression_eta: 0.0762 | Regression_pt: 0.2687 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.7690 | physical_nopu: 4.6810 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2125 | Classification_binary: 3.8371 | Regression_cos_phi: 1.9596 | Regression_energy: 1.5959 | Regression_eta: 1.0441 | Regression_pt: 1.8101 | Regression_sin_phi: 2.1345 | LR: 3.55e-04 | DataLoad Time: 0.0097s | Model Forward Time: 4.1190s | Phase Times: max_pre_step_barrier: 0.7590 | max_batch_to_device: 0.0007 | max_forward: 0.0763 | max_loss: 0.0189 | max_backward: 0.1117 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 2.0913 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5325.2 | uss_MB: 4459.8 | child_rss_MB: 9460.6 | allocated_MB: 394.0 | reserved_MB: 12220.0 | inactive_split_MB: 178.1 +rank1 [2026-10-27 17:33:22,189] INFO : Step 24000/100000 rank1 | Train Loss: 15.4786 | Train Loss Unweighted: 4.9908 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.5301 | Regression_cos_phi: 0.0467 | Regression_energy: 0.6682 | Regression_eta: 0.0624 | Regression_pt: 0.6223 | Regression_sin_phi: 0.0450 | Group Train Loss Unweighted (window mean): physical_pu: 3.7564 | physical_nopu: 4.6761 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4184 | Classification_binary: 3.4708 | Regression_cos_phi: 1.5869 | Regression_energy: 3.3386 | Regression_eta: 0.6141 | Regression_pt: 4.5342 | Regression_sin_phi: 1.4042 | LR: 3.51e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.9314s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0719 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1672 | max_tensorboard_logging: 1.8279 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5318.8 | uss_MB: 4462.5 | child_rss_MB: 9451.5 | allocated_MB: 393.8 | reserved_MB: 12220.0 | inactive_split_MB: 105.1 +rank1 [2026-10-27 17:36:26,064] INFO : Step 25000/100000 rank1 | Train Loss: 12.1246 | Train Loss Unweighted: 4.4783 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.7007 | Regression_cos_phi: 0.0551 | Regression_energy: 0.3196 | Regression_eta: 0.0694 | Regression_pt: 0.2642 | Regression_sin_phi: 0.0578 | Group Train Loss Unweighted (window mean): physical_pu: 3.7531 | physical_nopu: 4.6582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3013 | Classification_binary: 3.7589 | Regression_cos_phi: 2.3565 | Regression_energy: 1.8893 | Regression_eta: 0.8443 | Regression_pt: 2.3971 | Regression_sin_phi: 2.3482 | LR: 3.47e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.0306s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.1917 | max_loss: 0.0039 | max_backward: 0.1029 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 5304.8 | uss_MB: 4464.1 | child_rss_MB: 9445.8 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 86.4 +rank1 [2026-10-27 17:39:30,100] INFO : Step 26000/100000 rank1 | Train Loss: 12.6920 | Train Loss Unweighted: 4.5479 | Train Loss Components: Classification: 0.0188 | Classification_binary: 3.6003 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4034 | Regression_eta: 0.0734 | Regression_pt: 0.3564 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.7511 | physical_nopu: 4.6505 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3823 | Classification_binary: 3.7126 | Regression_cos_phi: 1.5205 | Regression_energy: 1.9508 | Regression_eta: 0.7793 | Regression_pt: 2.6025 | Regression_sin_phi: 1.6559 | LR: 3.43e-04 | DataLoad Time: 0.0014s | Model Forward Time: 3.0944s | Phase Times: max_pre_step_barrier: 0.0479 | max_batch_to_device: 0.0008 | max_forward: 0.1278 | max_loss: 0.0037 | max_backward: 0.0877 | max_optimizer_step: 0.1868 | max_tensorboard_logging: 1.8908 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5292.2 | uss_MB: 4467.2 | child_rss_MB: 9407.7 | allocated_MB: 395.0 | reserved_MB: 12220.0 | inactive_split_MB: 115.0 +rank1 [2026-10-27 17:42:32,922] INFO : Step 27000/100000 rank1 | Train Loss: 16.2758 | Train Loss Unweighted: 5.1421 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5608 | Regression_cos_phi: 0.0448 | Regression_energy: 0.7180 | Regression_eta: 0.0780 | Regression_pt: 0.6791 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.7443 | physical_nopu: 4.6426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3437 | Classification_binary: 3.7222 | Regression_cos_phi: 2.0246 | Regression_energy: 2.4806 | Regression_eta: 0.8574 | Regression_pt: 3.2505 | Regression_sin_phi: 1.7954 | LR: 3.39e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.0079s | Phase Times: max_pre_step_barrier: 0.0385 | max_batch_to_device: 0.0009 | max_forward: 0.0708 | max_loss: 0.0038 | max_backward: 0.0914 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.8878 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5283.2 | uss_MB: 4469.8 | child_rss_MB: 9400.6 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 87.9 +rank1 [2026-10-27 17:45:38,004] INFO : Step 28000/100000 rank1 | Train Loss: 11.9115 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6175 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3339 | Regression_eta: 0.0761 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.7393 | physical_nopu: 4.6288 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2692 | Classification_binary: 3.7460 | Regression_cos_phi: 1.9884 | Regression_energy: 1.5941 | Regression_eta: 0.7820 | Regression_pt: 1.9954 | Regression_sin_phi: 2.1105 | LR: 3.35e-04 | DataLoad Time: 0.0013s | Model Forward Time: 4.8029s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0008 | max_forward: 0.0886 | max_loss: 0.0038 | max_backward: 0.1193 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 3.6285 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 5274.2 | uss_MB: 4146.2 | child_rss_MB: 9355.1 | allocated_MB: 394.0 | reserved_MB: 12414.0 | inactive_split_MB: 106.5 +rank1 [2026-10-27 17:48:54,715] INFO : Step 29000/100000 rank1 | Train Loss: 13.9221 | Train Loss Unweighted: 4.6615 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.5075 | Regression_cos_phi: 0.0491 | Regression_energy: 0.5008 | Regression_eta: 0.0786 | Regression_pt: 0.4589 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7405 | physical_nopu: 4.6233 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3588 | Classification_binary: 3.4294 | Regression_cos_phi: 1.5758 | Regression_energy: 2.7591 | Regression_eta: 0.8628 | Regression_pt: 3.7557 | Regression_sin_phi: 1.5996 | LR: 3.30e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.3815s | Phase Times: max_pre_step_barrier: 0.1677 | max_batch_to_device: 0.0009 | max_forward: 0.0754 | max_loss: 0.0038 | max_backward: 0.0825 | max_optimizer_step: 0.1673 | max_tensorboard_logging: 2.1272 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5252.2 | uss_MB: 4453.2 | child_rss_MB: 9260.7 | allocated_MB: 394.9 | reserved_MB: 12414.0 | inactive_split_MB: 107.1 +rank1 [2026-10-27 17:51:57,834] INFO : Step 30000/100000 rank1 | Train Loss: 12.4988 | Train Loss Unweighted: 4.5639 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.6431 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3981 | Regression_eta: 0.0722 | Regression_pt: 0.3405 | Regression_sin_phi: 0.0498 | Group Train Loss Unweighted (window mean): physical_pu: 3.7342 | physical_nopu: 4.6125 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2537 | Classification_binary: 3.7136 | Regression_cos_phi: 1.6674 | Regression_energy: 1.8911 | Regression_eta: 0.9434 | Regression_pt: 2.3238 | Regression_sin_phi: 1.5337 | LR: 3.26e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.4591s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0762 | max_loss: 0.0136 | max_backward: 0.0854 | max_optimizer_step: 0.1722 | max_tensorboard_logging: 1.3349 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5242.2 | uss_MB: 4475.0 | child_rss_MB: 9193.8 | allocated_MB: 393.7 | reserved_MB: 12480.0 | inactive_split_MB: 89.7 +rank1 [2026-10-27 17:51:58,780] INFO : Running validation on rank1 +rank1 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 17:53:53,313] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 17:53:53,721] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 17:54:18,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 17:54:18,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 17:54:19,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 17:54:19,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 17:54:20,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 17:54:20,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 17:54:21,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 17:54:21,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 17:54:31,740] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 17:54:31,840] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 17:54:32,213] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 17:54:47,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 17:54:47,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 17:54:48,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 17:54:48,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 17:54:49,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 17:54:49,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 17:54:49,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 17:54:50,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 17:55:00,480] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 17:55:00,637] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 17:55:01,039] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 17:55:16,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 17:55:16,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 17:55:17,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 17:55:17,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 17:55:18,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 17:55:18,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 17:55:19,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 17:55:19,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 17:55:29,561] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 17:58:47,285] INFO : Step 31000/100000 rank1 | Train Loss: 12.0027 | Train Loss Unweighted: 4.4503 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.5779 | Regression_cos_phi: 0.0460 | Regression_energy: 0.3731 | Regression_eta: 0.0704 | Regression_pt: 0.3230 | Regression_sin_phi: 0.0451 | Group Train Loss Unweighted (window mean): physical_pu: 3.7262 | physical_nopu: 4.6083 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3449 | Classification_binary: 3.5218 | Regression_cos_phi: 1.6713 | Regression_energy: 2.3169 | Regression_eta: 1.1552 | Regression_pt: 2.6084 | Regression_sin_phi: 1.5410 | LR: 3.21e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.5860s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.1458 | max_loss: 0.0037 | max_backward: 0.1256 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 2.3030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7018.1 | uss_MB: 6249.8 | child_rss_MB: 9197.9 | allocated_MB: 394.8 | reserved_MB: 11168.0 | inactive_split_MB: 96.2 +rank1 [2026-10-27 18:01:54,932] INFO : Step 32000/100000 rank1 | Train Loss: 13.5217 | Train Loss Unweighted: 4.6918 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.5357 | Regression_cos_phi: 0.0443 | Regression_energy: 0.5169 | Regression_eta: 0.0728 | Regression_pt: 0.4649 | Regression_sin_phi: 0.0409 | Group Train Loss Unweighted (window mean): physical_pu: 3.7264 | physical_nopu: 4.6006 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3797 | Classification_binary: 3.3245 | Regression_cos_phi: 1.4436 | Regression_energy: 2.4668 | Regression_eta: 0.7813 | Regression_pt: 3.2119 | Regression_sin_phi: 1.1717 | LR: 3.16e-04 | DataLoad Time: 0.0072s | Model Forward Time: 2.3912s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.0795 | max_loss: 0.0917 | max_backward: 0.1851 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.1890 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5310.0 | uss_MB: 4562.1 | child_rss_MB: 9157.4 | allocated_MB: 394.9 | reserved_MB: 11240.0 | inactive_split_MB: 92.3 +rank1 [2026-10-27 18:04:57,678] INFO : Step 33000/100000 rank1 | Train Loss: 11.0126 | Train Loss Unweighted: 3.7465 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9928 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2905 | Regression_eta: 0.1058 | Regression_pt: 0.2472 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.7276 | physical_nopu: 4.5909 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3406 | Classification_binary: 3.0143 | Regression_cos_phi: 1.6489 | Regression_energy: 1.4466 | Regression_eta: 1.1502 | Regression_pt: 1.8145 | Regression_sin_phi: 1.6061 | LR: 3.12e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.8907s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0027 | max_forward: 0.1127 | max_loss: 0.0150 | max_backward: 0.3396 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.4660 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5295.8 | uss_MB: 4291.2 | child_rss_MB: 9057.9 | allocated_MB: 404.8 | reserved_MB: 11922.0 | inactive_split_MB: 135.4 +rank1 [2026-10-27 18:07:50,121] INFO : Step 34000/100000 rank1 | Train Loss: 14.2400 | Train Loss Unweighted: 4.7634 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5051 | Regression_cos_phi: 0.0450 | Regression_energy: 0.5627 | Regression_eta: 0.0631 | Regression_pt: 0.5251 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7180 | physical_nopu: 4.5858 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3653 | Classification_binary: 3.5936 | Regression_cos_phi: 1.7614 | Regression_energy: 2.9498 | Regression_eta: 0.7642 | Regression_pt: 3.8467 | Regression_sin_phi: 2.0993 | LR: 3.07e-04 | DataLoad Time: 0.0010s | Model Forward Time: 2.7416s | Phase Times: max_pre_step_barrier: 0.1802 | max_batch_to_device: 0.0765 | max_forward: 0.1650 | max_loss: 0.0036 | max_backward: 0.0935 | max_optimizer_step: 0.2788 | max_tensorboard_logging: 1.2621 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5285.5 | uss_MB: 4563.5 | child_rss_MB: 9059.3 | allocated_MB: 395.5 | reserved_MB: 11922.0 | inactive_split_MB: 99.3 +rank1 [2026-10-27 18:10:53,637] INFO : Step 35000/100000 rank1 | Train Loss: 11.4005 | Train Loss Unweighted: 4.4142 | Train Loss Components: Classification: 0.0091 | Classification_binary: 3.6972 | Regression_cos_phi: 0.0530 | Regression_energy: 0.2984 | Regression_eta: 0.0671 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.7147 | physical_nopu: 4.5802 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1640 | Classification_binary: 3.7608 | Regression_cos_phi: 1.9712 | Regression_energy: 1.4174 | Regression_eta: 1.0619 | Regression_pt: 1.5681 | Regression_sin_phi: 1.6578 | LR: 3.02e-04 | DataLoad Time: 0.0678s | Model Forward Time: 2.3501s | Phase Times: max_pre_step_barrier: 0.0682 | max_batch_to_device: 0.0008 | max_forward: 0.1243 | max_loss: 0.0038 | max_backward: 0.0883 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.1949 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5274.5 | uss_MB: 4566.4 | child_rss_MB: 9063.3 | allocated_MB: 393.8 | reserved_MB: 11922.0 | inactive_split_MB: 85.2 +rank1 [2026-10-27 18:13:50,577] INFO : Step 36000/100000 rank1 | Train Loss: 11.8438 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.5751 | Regression_cos_phi: 0.0431 | Regression_energy: 0.3555 | Regression_eta: 0.0641 | Regression_pt: 0.3210 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.7110 | physical_nopu: 4.5729 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3917 | Classification_binary: 3.7299 | Regression_cos_phi: 1.2979 | Regression_energy: 1.5495 | Regression_eta: 0.6109 | Regression_pt: 2.1237 | Regression_sin_phi: 1.4721 | LR: 2.97e-04 | DataLoad Time: 0.0007s | Model Forward Time: 2.4524s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0993 | max_loss: 0.0066 | max_backward: 0.1760 | max_optimizer_step: 0.1735 | max_tensorboard_logging: 1.2166 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 5259.5 | uss_MB: 4568.3 | child_rss_MB: 9032.9 | allocated_MB: 395.0 | reserved_MB: 11922.0 | inactive_split_MB: 95.2 +rank1 [2026-10-27 18:16:45,596] INFO : Step 37000/100000 rank1 | Train Loss: 15.1390 | Train Loss Unweighted: 4.8166 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4396 | Regression_cos_phi: 0.0489 | Regression_energy: 0.6142 | Regression_eta: 0.0886 | Regression_pt: 0.5621 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7102 | physical_nopu: 4.5673 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3515 | Classification_binary: 3.4789 | Regression_cos_phi: 1.2980 | Regression_energy: 2.4852 | Regression_eta: 0.9050 | Regression_pt: 3.1355 | Regression_sin_phi: 1.1606 | LR: 2.91e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.2788s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1889 | max_loss: 0.0036 | max_backward: 0.0899 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 1.0901 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 5243.2 | uss_MB: 4569.9 | child_rss_MB: 9022.4 | allocated_MB: 394.3 | reserved_MB: 11922.0 | inactive_split_MB: 107.1 +rank1 [2026-10-27 18:19:42,075] INFO : Step 38000/100000 rank1 | Train Loss: 11.6256 | Train Loss Unweighted: 4.4424 | Train Loss Components: Classification: 0.0117 | Classification_binary: 3.6501 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3369 | Regression_eta: 0.0664 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7131 | physical_nopu: 4.5682 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2939 | Classification_binary: 3.5335 | Regression_cos_phi: 1.5785 | Regression_energy: 1.7407 | Regression_eta: 0.6258 | Regression_pt: 2.0884 | Regression_sin_phi: 1.5341 | LR: 2.86e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.2499s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.0965 | max_loss: 0.0038 | max_backward: 0.1051 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5237.2 | uss_MB: 4571.0 | child_rss_MB: 9030.3 | allocated_MB: 393.3 | reserved_MB: 12082.0 | inactive_split_MB: 110.5 +rank1 [2026-10-27 18:22:37,812] INFO : Step 39000/100000 rank1 | Train Loss: 13.0208 | Train Loss Unweighted: 4.4928 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4758 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4360 | Regression_eta: 0.0793 | Regression_pt: 0.3879 | Regression_sin_phi: 0.0500 | Group Train Loss Unweighted (window mean): physical_pu: 3.7060 | physical_nopu: 4.5607 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3495 | Classification_binary: 3.4203 | Regression_cos_phi: 1.8882 | Regression_energy: 1.6898 | Regression_eta: 0.7104 | Regression_pt: 2.2113 | Regression_sin_phi: 1.6107 | LR: 2.81e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.2312s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.0864 | max_loss: 0.0038 | max_backward: 0.0877 | max_optimizer_step: 0.1700 | max_tensorboard_logging: 1.1418 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5224.7 | uss_MB: 4572.8 | child_rss_MB: 9008.2 | allocated_MB: 394.4 | reserved_MB: 12706.0 | inactive_split_MB: 84.9 +rank1 [2026-10-27 18:25:31,277] INFO : Step 40000/100000 rank1 | Train Loss: 14.7562 | Train Loss Unweighted: 4.9007 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5708 | Regression_cos_phi: 0.0459 | Regression_energy: 0.5954 | Regression_eta: 0.0747 | Regression_pt: 0.5553 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7006 | physical_nopu: 4.5579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2591 | Classification_binary: 3.7244 | Regression_cos_phi: 1.7425 | Regression_energy: 3.0626 | Regression_eta: 0.8753 | Regression_pt: 4.1334 | Regression_sin_phi: 1.9747 | LR: 2.76e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.3074s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.0814 | max_loss: 0.0614 | max_backward: 0.1417 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 1.1378 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5222.7 | uss_MB: 4573.6 | child_rss_MB: 8992.8 | allocated_MB: 393.7 | reserved_MB: 12706.0 | inactive_split_MB: 107.7 +rank1 [2026-10-27 18:25:32,320] INFO : Running validation on rank1 +rank1 [2026-10-27 18:25:32,320] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 18:27:23,860] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 18:27:24,220] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 18:27:48,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 18:27:48,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 18:27:49,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 18:27:49,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 18:27:50,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 18:27:50,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 18:27:51,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 18:27:51,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 18:28:01,625] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 18:28:01,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 18:28:02,235] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 18:28:17,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 18:28:17,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 18:28:18,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 18:28:18,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 18:28:19,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 18:28:19,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 18:28:20,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 18:28:20,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 18:28:30,525] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 18:28:30,988] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 18:28:31,367] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 18:28:46,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 18:28:47,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 18:28:47,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 18:28:48,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 18:28:48,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 18:28:48,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 18:28:49,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 18:28:49,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 18:28:59,787] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 18:32:27,442] INFO : Step 41000/100000 rank1 | Train Loss: 12.3609 | Train Loss Unweighted: 4.4817 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5838 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3837 | Regression_eta: 0.0731 | Regression_pt: 0.3287 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.7016 | physical_nopu: 4.5433 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2710 | Classification_binary: 3.6483 | Regression_cos_phi: 1.6392 | Regression_energy: 1.7159 | Regression_eta: 1.2876 | Regression_pt: 2.0678 | Regression_sin_phi: 1.5261 | LR: 2.70e-04 | DataLoad Time: 0.0022s | Model Forward Time: 4.5170s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0009 | max_forward: 0.1338 | max_loss: 0.0038 | max_backward: 0.1857 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 3.1120 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 7028.8 | uss_MB: 6392.1 | child_rss_MB: 8998.3 | allocated_MB: 394.0 | reserved_MB: 11364.0 | inactive_split_MB: 106.1 +rank1 [2026-10-27 18:35:19,857] INFO : Step 42000/100000 rank1 | Train Loss: 12.9790 | Train Loss Unweighted: 4.5354 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4379 | Regression_cos_phi: 0.0458 | Regression_energy: 0.4856 | Regression_eta: 0.0625 | Regression_pt: 0.4500 | Regression_sin_phi: 0.0370 | Group Train Loss Unweighted (window mean): physical_pu: 3.6959 | physical_nopu: 4.5474 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3593 | Classification_binary: 3.4435 | Regression_cos_phi: 1.4759 | Regression_energy: 2.7940 | Regression_eta: 0.5329 | Regression_pt: 4.0017 | Regression_sin_phi: 1.0253 | LR: 2.65e-04 | DataLoad Time: 0.0112s | Model Forward Time: 2.6327s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0009 | max_forward: 0.1375 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 1.3492 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 7016.8 | uss_MB: 5332.4 | child_rss_MB: 8994.4 | allocated_MB: 394.2 | reserved_MB: 12150.0 | inactive_split_MB: 99.3 +rank1 [2026-10-27 18:38:35,925] INFO : Step 43000/100000 rank1 | Train Loss: 10.9006 | Train Loss Unweighted: 3.7057 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.9740 | Regression_cos_phi: 0.0473 | Regression_energy: 0.2786 | Regression_eta: 0.1094 | Regression_pt: 0.2322 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6942 | physical_nopu: 4.5421 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3725 | Classification_binary: 2.9106 | Regression_cos_phi: 1.6775 | Regression_energy: 1.3403 | Regression_eta: 1.2865 | Regression_pt: 1.6271 | Regression_sin_phi: 1.6645 | LR: 2.59e-04 | DataLoad Time: 0.0018s | Model Forward Time: 5.2293s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0039 | max_forward: 0.1100 | max_loss: 0.0637 | max_backward: 0.2303 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 3.9250 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5285.0 | uss_MB: 4686.1 | child_rss_MB: 8875.5 | allocated_MB: 406.8 | reserved_MB: 12150.0 | inactive_split_MB: 152.7 +rank1 [2026-10-27 18:41:39,775] INFO : Step 44000/100000 rank1 | Train Loss: 14.8629 | Train Loss Unweighted: 4.8399 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4856 | Regression_cos_phi: 0.0438 | Regression_energy: 0.6070 | Regression_eta: 0.0739 | Regression_pt: 0.5644 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6933 | physical_nopu: 4.5331 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3605 | Classification_binary: 3.7899 | Regression_cos_phi: 1.8479 | Regression_energy: 2.8842 | Regression_eta: 0.8047 | Regression_pt: 3.9216 | Regression_sin_phi: 1.8782 | LR: 2.54e-04 | DataLoad Time: 0.0844s | Model Forward Time: 2.6013s | Phase Times: max_pre_step_barrier: 0.0851 | max_batch_to_device: 0.0008 | max_forward: 0.0946 | max_loss: 0.0041 | max_backward: 0.0964 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.3558 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5284.0 | uss_MB: 4679.1 | child_rss_MB: 8885.5 | allocated_MB: 394.8 | reserved_MB: 12150.0 | inactive_split_MB: 94.9 +rank1 [2026-10-27 18:44:46,984] INFO : Step 45000/100000 rank1 | Train Loss: 12.9535 | Train Loss Unweighted: 4.5658 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.6115 | Regression_cos_phi: 0.0537 | Regression_energy: 0.4117 | Regression_eta: 0.0846 | Regression_pt: 0.3413 | Regression_sin_phi: 0.0531 | Group Train Loss Unweighted (window mean): physical_pu: 3.6892 | physical_nopu: 4.5283 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2223 | Classification_binary: 3.6288 | Regression_cos_phi: 1.8488 | Regression_energy: 1.5996 | Regression_eta: 0.7099 | Regression_pt: 2.0326 | Regression_sin_phi: 1.5756 | LR: 2.48e-04 | DataLoad Time: 0.0029s | Model Forward Time: 2.4259s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1077 | max_loss: 0.0040 | max_backward: 0.1457 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 1.2404 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 5275.4 | uss_MB: 4680.1 | child_rss_MB: 8893.2 | allocated_MB: 394.0 | reserved_MB: 13366.0 | inactive_split_MB: 74.2 +rank1 [2026-10-27 18:48:06,241] INFO : Step 46000/100000 rank1 | Train Loss: 12.3783 | Train Loss Unweighted: 4.4476 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.5252 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3923 | Regression_eta: 0.0679 | Regression_pt: 0.3506 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6869 | physical_nopu: 4.5281 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4512 | Classification_binary: 3.4396 | Regression_cos_phi: 2.0861 | Regression_energy: 1.9020 | Regression_eta: 0.5535 | Regression_pt: 2.6059 | Regression_sin_phi: 1.3287 | LR: 2.43e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.4376s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1021 | max_loss: 0.0036 | max_backward: 0.1895 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 1.1894 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 5269.4 | uss_MB: 4681.7 | child_rss_MB: 8897.5 | allocated_MB: 395.1 | reserved_MB: 13366.0 | inactive_split_MB: 103.1 +rank1 [2026-10-27 18:51:16,142] INFO : Step 47000/100000 rank1 | Train Loss: 14.5142 | Train Loss Unweighted: 4.7631 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.3922 | Regression_cos_phi: 0.0407 | Regression_energy: 0.6220 | Regression_eta: 0.0662 | Regression_pt: 0.5862 | Regression_sin_phi: 0.0406 | Group Train Loss Unweighted (window mean): physical_pu: 3.6853 | physical_nopu: 4.5234 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3602 | Classification_binary: 3.4302 | Regression_cos_phi: 1.4165 | Regression_energy: 2.5347 | Regression_eta: 0.6145 | Regression_pt: 3.4154 | Regression_sin_phi: 1.1660 | LR: 2.37e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.5374s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0667 | max_forward: 0.1426 | max_loss: 0.0037 | max_backward: 0.0873 | max_optimizer_step: 0.2591 | max_tensorboard_logging: 1.1650 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5267.4 | uss_MB: 4682.3 | child_rss_MB: 8896.3 | allocated_MB: 394.9 | reserved_MB: 14166.0 | inactive_split_MB: 95.2 +rank1 [2026-10-27 18:54:12,871] INFO : Step 48000/100000 rank1 | Train Loss: 11.2030 | Train Loss Unweighted: 4.3343 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.6447 | Regression_cos_phi: 0.0521 | Regression_energy: 0.2822 | Regression_eta: 0.0657 | Regression_pt: 0.2293 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6844 | physical_nopu: 4.5168 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2667 | Classification_binary: 3.4849 | Regression_cos_phi: 1.8619 | Regression_energy: 1.2037 | Regression_eta: 0.5705 | Regression_pt: 1.4215 | Regression_sin_phi: 1.4170 | LR: 2.31e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.2609s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.0058 | max_backward: 0.0852 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1613 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5253.4 | uss_MB: 4683.3 | child_rss_MB: 8890.9 | allocated_MB: 393.9 | reserved_MB: 14166.0 | inactive_split_MB: 101.2 +rank1 [2026-10-27 18:57:19,782] INFO : Step 49000/100000 rank1 | Train Loss: 12.4372 | Train Loss Unweighted: 4.4483 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.5320 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3881 | Regression_eta: 0.0742 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6801 | physical_nopu: 4.5174 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3697 | Classification_binary: 3.4102 | Regression_cos_phi: 2.2142 | Regression_energy: 2.0446 | Regression_eta: 0.6876 | Regression_pt: 2.5459 | Regression_sin_phi: 1.7228 | LR: 2.26e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.4738s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0009 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1014 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 1.3340 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5240.2 | uss_MB: 4683.4 | child_rss_MB: 8815.5 | allocated_MB: 393.7 | reserved_MB: 14166.0 | inactive_split_MB: 100.8 +rank1 [2026-10-27 19:00:32,233] INFO : Step 50000/100000 rank1 | Train Loss: 13.3654 | Train Loss Unweighted: 4.5742 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4424 | Regression_cos_phi: 0.0426 | Regression_energy: 0.5013 | Regression_eta: 0.0683 | Regression_pt: 0.4573 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6814 | physical_nopu: 4.5132 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2423 | Classification_binary: 3.5468 | Regression_cos_phi: 1.8922 | Regression_energy: 1.7490 | Regression_eta: 0.8433 | Regression_pt: 2.1683 | Regression_sin_phi: 1.5224 | LR: 2.20e-04 | DataLoad Time: 0.0012s | Model Forward Time: 2.5867s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0010 | max_forward: 0.1892 | max_loss: 0.0115 | max_backward: 0.1333 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.2641 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0862 | Memory: rss_MB: 5234.2 | uss_MB: 4685.5 | child_rss_MB: 8779.3 | allocated_MB: 394.6 | reserved_MB: 14166.0 | inactive_split_MB: 91.2 +rank1 [2026-10-27 19:00:33,174] INFO : Running validation on rank1 +rank1 [2026-10-27 19:00:33,174] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 19:00:33,174] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 19:00:33,174] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 19:00:33,174] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 19:02:37,501] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 19:02:37,940] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 19:03:02,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 19:03:03,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 19:03:03,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 19:03:04,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 19:03:04,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 19:03:05,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 19:03:05,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 19:03:06,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 19:03:16,033] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 19:03:16,262] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 19:03:16,635] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 19:03:31,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 19:03:32,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 19:03:32,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 19:03:33,354] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 19:03:33,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 19:03:34,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 19:03:34,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 19:03:35,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 19:03:45,242] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 19:03:45,438] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 19:03:45,816] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 19:04:00,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 19:04:01,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 19:04:01,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 19:04:02,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 19:04:02,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 19:04:02,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 19:04:03,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 19:04:03,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 19:04:13,871] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 19:07:44,423] INFO : Step 51000/100000 rank1 | Train Loss: 12.0993 | Train Loss Unweighted: 4.3946 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5485 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3591 | Regression_eta: 0.0789 | Regression_pt: 0.2941 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6745 | physical_nopu: 4.5037 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2347 | Classification_binary: 3.8333 | Regression_cos_phi: 1.8709 | Regression_energy: 1.3838 | Regression_eta: 0.8056 | Regression_pt: 1.5893 | Regression_sin_phi: 1.4450 | LR: 2.14e-04 | DataLoad Time: 0.0097s | Model Forward Time: 3.3976s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0008 | max_forward: 0.1391 | max_loss: 0.0036 | max_backward: 0.1509 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 2.0683 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 7102.6 | uss_MB: 6555.6 | child_rss_MB: 8775.1 | allocated_MB: 393.6 | reserved_MB: 10952.0 | inactive_split_MB: 101.2 +rank1 [2026-10-27 19:10:55,345] INFO : Step 52000/100000 rank1 | Train Loss: 10.7195 | Train Loss Unweighted: 3.6688 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9553 | Regression_cos_phi: 0.0470 | Regression_energy: 0.2719 | Regression_eta: 0.1059 | Regression_pt: 0.2258 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6784 | physical_nopu: 4.5078 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3562 | Classification_binary: 2.8695 | Regression_cos_phi: 1.7178 | Regression_energy: 1.3312 | Regression_eta: 1.1466 | Regression_pt: 1.6371 | Regression_sin_phi: 1.5247 | LR: 2.09e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.5036s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0031 | max_forward: 0.1853 | max_loss: 0.0256 | max_backward: 0.2604 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 2.0239 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7092.6 | uss_MB: 6564.5 | child_rss_MB: 8773.5 | allocated_MB: 404.8 | reserved_MB: 11626.0 | inactive_split_MB: 256.4 +rank1 [2026-10-27 19:13:48,854] INFO : Step 53000/100000 rank1 | Train Loss: 11.7022 | Train Loss Unweighted: 4.3546 | Train Loss Components: Classification: 0.0123 | Classification_binary: 3.5594 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3321 | Regression_eta: 0.0730 | Regression_pt: 0.2795 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6748 | physical_nopu: 4.5005 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2412 | Classification_binary: 3.7120 | Regression_cos_phi: 1.8266 | Regression_energy: 1.5190 | Regression_eta: 0.5557 | Regression_pt: 1.8968 | Regression_sin_phi: 1.5049 | LR: 2.03e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.1500s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0954 | max_loss: 0.0037 | max_backward: 0.1065 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.9892 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 5255.7 | uss_MB: 4729.4 | child_rss_MB: 8773.6 | allocated_MB: 393.6 | reserved_MB: 11910.0 | inactive_split_MB: 100.6 +rank1 [2026-10-27 19:17:18,773] INFO : Step 54000/100000 rank1 | Train Loss: 12.5762 | Train Loss Unweighted: 4.3665 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0487 | Regression_energy: 0.4104 | Regression_eta: 0.0739 | Regression_pt: 0.3685 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6696 | physical_nopu: 4.5036 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3280 | Classification_binary: 3.3270 | Regression_cos_phi: 1.5663 | Regression_energy: 1.6507 | Regression_eta: 0.6785 | Regression_pt: 2.2021 | Regression_sin_phi: 1.5681 | LR: 1.97e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.3959s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.1022 | max_forward: 0.1953 | max_loss: 0.0037 | max_backward: 0.0975 | max_optimizer_step: 0.2772 | max_tensorboard_logging: 2.0527 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5248.7 | uss_MB: 4730.3 | child_rss_MB: 8744.8 | allocated_MB: 394.9 | reserved_MB: 11910.0 | inactive_split_MB: 118.2 +rank1 [2026-10-27 19:20:28,222] INFO : Step 55000/100000 rank1 | Train Loss: 13.4632 | Train Loss Unweighted: 4.6500 | Train Loss Components: Classification: 0.0134 | Classification_binary: 3.5428 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4921 | Regression_eta: 0.0679 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6720 | physical_nopu: 4.4919 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2301 | Classification_binary: 3.4993 | Regression_cos_phi: 1.4326 | Regression_energy: 2.5632 | Regression_eta: 0.6146 | Regression_pt: 3.4027 | Regression_sin_phi: 1.3491 | LR: 1.92e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.8021s | Phase Times: max_pre_step_barrier: 0.1980 | max_batch_to_device: 0.0007 | max_forward: 0.0889 | max_loss: 0.0037 | max_backward: 0.2091 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 2.3779 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5235.8 | uss_MB: 4729.6 | child_rss_MB: 8640.7 | allocated_MB: 393.8 | reserved_MB: 11910.0 | inactive_split_MB: 89.1 +rank1 [2026-10-27 19:23:30,605] INFO : Step 56000/100000 rank1 | Train Loss: 11.6078 | Train Loss Unweighted: 4.2962 | Train Loss Components: Classification: 0.0140 | Classification_binary: 3.4759 | Regression_cos_phi: 0.0438 | Regression_energy: 0.3484 | Regression_eta: 0.0671 | Regression_pt: 0.2977 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6692 | physical_nopu: 4.4908 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2390 | Classification_binary: 3.3740 | Regression_cos_phi: 1.2015 | Regression_energy: 1.8596 | Regression_eta: 0.6737 | Regression_pt: 2.3459 | Regression_sin_phi: 1.6381 | LR: 1.86e-04 | DataLoad Time: 0.0520s | Model Forward Time: 3.1794s | Phase Times: max_pre_step_barrier: 0.0941 | max_batch_to_device: 0.0008 | max_forward: 0.0930 | max_loss: 0.0038 | max_backward: 0.0988 | max_optimizer_step: 0.1756 | max_tensorboard_logging: 1.8920 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 5221.8 | uss_MB: 4732.9 | child_rss_MB: 8582.9 | allocated_MB: 393.5 | reserved_MB: 11910.0 | inactive_split_MB: 115.7 +rank1 [2026-10-27 19:26:52,902] INFO : Step 57000/100000 rank1 | Train Loss: 13.9252 | Train Loss Unweighted: 4.6468 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4640 | Regression_cos_phi: 0.0464 | Regression_energy: 0.5144 | Regression_eta: 0.0781 | Regression_pt: 0.4791 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6687 | physical_nopu: 4.4875 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4028 | Classification_binary: 3.5832 | Regression_cos_phi: 1.4521 | Regression_energy: 2.6375 | Regression_eta: 0.6157 | Regression_pt: 3.5860 | Regression_sin_phi: 1.9861 | LR: 1.81e-04 | DataLoad Time: 0.0023s | Model Forward Time: 4.6260s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.1223 | max_forward: 0.2094 | max_loss: 0.0420 | max_backward: 0.1628 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 2.9072 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 5216.8 | uss_MB: 4734.2 | child_rss_MB: 8511.2 | allocated_MB: 394.3 | reserved_MB: 12240.0 | inactive_split_MB: 101.4 +rank1 [2026-10-27 19:30:07,651] INFO : Step 58000/100000 rank1 | Train Loss: 11.5910 | Train Loss Unweighted: 4.3047 | Train Loss Components: Classification: 0.0106 | Classification_binary: 3.5490 | Regression_cos_phi: 0.0544 | Regression_energy: 0.3108 | Regression_eta: 0.0631 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0510 | Group Train Loss Unweighted (window mean): physical_pu: 3.6630 | physical_nopu: 4.4842 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1966 | Classification_binary: 3.5352 | Regression_cos_phi: 2.0526 | Regression_energy: 1.2410 | Regression_eta: 0.6082 | Regression_pt: 1.5454 | Regression_sin_phi: 1.4472 | LR: 1.75e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.5122s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.1080 | max_backward: 0.1964 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.2749 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 5209.3 | uss_MB: 4734.4 | child_rss_MB: 8432.6 | allocated_MB: 393.9 | reserved_MB: 12240.0 | inactive_split_MB: 121.9 +rank1 [2026-10-27 19:33:12,149] INFO : Step 59000/100000 rank1 | Train Loss: 12.2218 | Train Loss Unweighted: 4.4115 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5166 | Regression_cos_phi: 0.0458 | Regression_energy: 0.3794 | Regression_eta: 0.0771 | Regression_pt: 0.3280 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6650 | physical_nopu: 4.4788 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3616 | Classification_binary: 3.5247 | Regression_cos_phi: 1.5651 | Regression_energy: 1.6985 | Regression_eta: 1.4474 | Regression_pt: 2.0253 | Regression_sin_phi: 1.6987 | LR: 1.70e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.2206s | Phase Times: max_pre_step_barrier: 0.6889 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0499 | max_backward: 0.1549 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5201.3 | uss_MB: 4735.4 | child_rss_MB: 8387.3 | allocated_MB: 394.0 | reserved_MB: 12240.0 | inactive_split_MB: 135.3 +rank1 [2026-10-27 19:36:26,517] INFO : Step 60000/100000 rank1 | Train Loss: 14.3810 | Train Loss Unweighted: 4.7229 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.4146 | Regression_cos_phi: 0.0449 | Regression_energy: 0.5841 | Regression_eta: 0.0716 | Regression_pt: 0.5495 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.6604 | physical_nopu: 4.4865 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3395 | Classification_binary: 3.4848 | Regression_cos_phi: 1.7180 | Regression_energy: 2.3885 | Regression_eta: 0.9670 | Regression_pt: 3.1419 | Regression_sin_phi: 1.5283 | LR: 1.64e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.4566s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0849 | max_loss: 0.0036 | max_backward: 0.1478 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 1.2880 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5191.9 | uss_MB: 4495.0 | child_rss_MB: 8328.9 | allocated_MB: 394.8 | reserved_MB: 12240.0 | inactive_split_MB: 87.0 +rank1 [2026-10-27 19:36:27,558] INFO : Running validation on rank1 +rank1 [2026-10-27 19:36:27,558] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 19:36:27,558] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 19:36:27,558] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 19:38:18,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 19:38:19,028] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 19:38:43,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 19:38:44,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 19:38:44,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 19:38:45,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 19:38:45,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 19:38:46,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 19:38:46,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 19:38:46,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 19:38:56,961] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 19:38:57,241] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 19:38:57,609] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 19:39:12,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 19:39:12,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 19:39:13,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 19:39:13,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 19:39:14,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 19:39:14,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 19:39:15,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 19:39:15,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 19:39:25,531] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 19:39:26,170] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 19:39:26,533] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 19:39:41,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 19:39:42,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 19:39:42,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 19:39:43,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 19:39:43,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 19:39:44,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 19:39:44,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 19:39:44,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 19:39:54,974] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-27 19:43:33,543] INFO : Step 61000/100000 rank1 | Train Loss: 11.4262 | Train Loss Unweighted: 4.3706 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.6500 | Regression_cos_phi: 0.0531 | Regression_energy: 0.2956 | Regression_eta: 0.0675 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6553 | physical_nopu: 4.4780 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2597 | Classification_binary: 3.5869 | Regression_cos_phi: 1.8394 | Regression_energy: 1.3672 | Regression_eta: 0.6718 | Regression_pt: 1.5639 | Regression_sin_phi: 1.5957 | LR: 1.59e-04 | DataLoad Time: 0.0077s | Model Forward Time: 3.1986s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1116 | max_loss: 0.0036 | max_backward: 0.1506 | max_optimizer_step: 0.1828 | max_tensorboard_logging: 1.9738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 5784.0 | uss_MB: 5329.2 | child_rss_MB: 8302.4 | allocated_MB: 393.5 | reserved_MB: 11190.0 | inactive_split_MB: 103.1 +rank1 [2026-10-27 19:46:49,589] INFO : Step 62000/100000 rank1 | Train Loss: 10.7781 | Train Loss Unweighted: 3.6510 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9253 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2791 | Regression_eta: 0.1043 | Regression_pt: 0.2316 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3503 | Classification_binary: 2.9470 | Regression_cos_phi: 1.7042 | Regression_energy: 1.3043 | Regression_eta: 1.2497 | Regression_pt: 1.5499 | Regression_sin_phi: 1.5583 | LR: 1.54e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.6559s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0030 | max_forward: 0.1294 | max_loss: 0.0038 | max_backward: 0.2007 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.2115 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0496 | Memory: rss_MB: 5775.0 | uss_MB: 5052.2 | child_rss_MB: 8186.6 | allocated_MB: 404.4 | reserved_MB: 11190.0 | inactive_split_MB: 148.6 +rank1 [2026-10-27 19:50:02,443] INFO : Step 63000/100000 rank1 | Train Loss: 11.5766 | Train Loss Unweighted: 4.3655 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6000 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3204 | Regression_eta: 0.0727 | Regression_pt: 0.2610 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6589 | physical_nopu: 4.4771 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2744 | Classification_binary: 3.5664 | Regression_cos_phi: 1.7308 | Regression_energy: 1.3428 | Regression_eta: 0.6526 | Regression_pt: 1.6388 | Regression_sin_phi: 1.6070 | LR: 1.49e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.3897s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.1036 | max_loss: 0.0037 | max_backward: 0.1140 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2208 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 5773.0 | uss_MB: 5331.6 | child_rss_MB: 8138.8 | allocated_MB: 393.8 | reserved_MB: 11190.0 | inactive_split_MB: 93.0 +rank1 [2026-10-27 19:53:17,771] INFO : Step 64000/100000 rank1 | Train Loss: 13.2709 | Train Loss Unweighted: 4.5542 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.4295 | Regression_cos_phi: 0.0429 | Regression_energy: 0.4948 | Regression_eta: 0.0672 | Regression_pt: 0.4587 | Regression_sin_phi: 0.0436 | Group Train Loss Unweighted (window mean): physical_pu: 3.6551 | physical_nopu: 4.4714 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3619 | Classification_binary: 3.6538 | Regression_cos_phi: 1.6756 | Regression_energy: 2.9473 | Regression_eta: 0.6755 | Regression_pt: 4.1311 | Regression_sin_phi: 1.3119 | LR: 1.43e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.3961s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0928 | max_loss: 0.0570 | max_backward: 0.1589 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 1.2035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 5289.6 | uss_MB: 4857.6 | child_rss_MB: 8120.4 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank1 [2026-10-27 19:56:23,053] INFO : Step 65000/100000 rank1 | Train Loss: 15.6652 | Train Loss Unweighted: 4.9833 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5112 | Regression_cos_phi: 0.0496 | Regression_energy: 0.6688 | Regression_eta: 0.0629 | Regression_pt: 0.6290 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6554 | physical_nopu: 4.4650 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3193 | Classification_binary: 3.6461 | Regression_cos_phi: 1.4175 | Regression_energy: 2.6034 | Regression_eta: 0.5744 | Regression_pt: 3.5180 | Regression_sin_phi: 1.4462 | LR: 1.38e-04 | DataLoad Time: 0.0883s | Model Forward Time: 2.6766s | Phase Times: max_pre_step_barrier: 0.1158 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0042 | max_backward: 0.1974 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.4176 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5282.6 | uss_MB: 4858.3 | child_rss_MB: 8025.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 124.9 +rank1 [2026-10-27 19:59:28,722] INFO : Step 66000/100000 rank1 | Train Loss: 12.0073 | Train Loss Unweighted: 4.3350 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4949 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3535 | Regression_eta: 0.0691 | Regression_pt: 0.3018 | Regression_sin_phi: 0.0501 | Group Train Loss Unweighted (window mean): physical_pu: 3.6526 | physical_nopu: 4.4657 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4100 | Classification_binary: 3.3916 | Regression_cos_phi: 1.5896 | Regression_energy: 2.5886 | Regression_eta: 1.0681 | Regression_pt: 3.1928 | Regression_sin_phi: 1.6497 | LR: 1.33e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.4469s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0891 | max_loss: 0.0538 | max_backward: 0.1468 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 1.2753 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5277.7 | uss_MB: 4858.9 | child_rss_MB: 7982.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 107.1 +rank1 [2026-10-27 20:02:51,910] INFO : Step 67000/100000 rank1 | Train Loss: 13.3177 | Train Loss Unweighted: 4.5554 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.4800 | Regression_cos_phi: 0.0500 | Regression_energy: 0.4690 | Regression_eta: 0.0673 | Regression_pt: 0.4253 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6538 | physical_nopu: 4.4665 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3454 | Classification_binary: 3.5303 | Regression_cos_phi: 1.4648 | Regression_energy: 2.6254 | Regression_eta: 0.7426 | Regression_pt: 3.5957 | Regression_sin_phi: 2.0575 | LR: 1.28e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.9379s | Phase Times: max_pre_step_barrier: 0.0792 | max_batch_to_device: 0.0009 | max_forward: 0.0886 | max_loss: 0.1057 | max_backward: 0.1987 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 2.6109 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 5271.7 | uss_MB: 4859.8 | child_rss_MB: 7952.8 | allocated_MB: 393.9 | reserved_MB: 11792.0 | inactive_split_MB: 111.5 +rank1 [2026-10-27 20:06:09,103] INFO : Step 68000/100000 rank1 | Train Loss: 11.1535 | Train Loss Unweighted: 4.2643 | Train Loss Components: Classification: 0.0109 | Classification_binary: 3.5138 | Regression_cos_phi: 0.0456 | Regression_energy: 0.3163 | Regression_eta: 0.0648 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6527 | physical_nopu: 4.4693 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3051 | Classification_binary: 3.4408 | Regression_cos_phi: 1.2102 | Regression_energy: 1.4610 | Regression_eta: 0.6687 | Regression_pt: 1.8116 | Regression_sin_phi: 1.2738 | LR: 1.24e-04 | DataLoad Time: 0.0015s | Model Forward Time: 2.3903s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1008 | max_loss: 0.0767 | max_backward: 0.1768 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.1759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5263.4 | uss_MB: 4860.3 | child_rss_MB: 7883.7 | allocated_MB: 394.5 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank1 [2026-10-27 20:09:12,492] INFO : Step 69000/100000 rank1 | Train Loss: 12.1728 | Train Loss Unweighted: 4.3418 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4719 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3660 | Regression_eta: 0.0763 | Regression_pt: 0.3115 | Regression_sin_phi: 0.0548 | Group Train Loss Unweighted (window mean): physical_pu: 3.6494 | physical_nopu: 4.4629 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3068 | Classification_binary: 3.4775 | Regression_cos_phi: 1.4763 | Regression_energy: 1.8581 | Regression_eta: 0.5368 | Regression_pt: 2.2353 | Regression_sin_phi: 1.6004 | LR: 1.19e-04 | DataLoad Time: 0.0008s | Model Forward Time: 2.2969s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.1094 | max_loss: 0.0147 | max_backward: 0.1167 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 1.1204 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5251.4 | uss_MB: 4861.5 | child_rss_MB: 7816.8 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 106.5 +rank1 [2026-10-27 20:12:13,498] INFO : Step 70000/100000 rank1 | Train Loss: 14.9156 | Train Loss Unweighted: 4.7730 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.3975 | Regression_cos_phi: 0.0487 | Regression_energy: 0.6183 | Regression_eta: 0.0638 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0448 | Group Train Loss Unweighted (window mean): physical_pu: 3.6488 | physical_nopu: 4.4584 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3781 | Classification_binary: 3.2688 | Regression_cos_phi: 1.3165 | Regression_energy: 1.8926 | Regression_eta: 0.7418 | Regression_pt: 2.5310 | Regression_sin_phi: 1.1393 | LR: 1.14e-04 | DataLoad Time: 0.0073s | Model Forward Time: 2.3862s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1038 | max_loss: 0.0858 | max_backward: 0.1871 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 1.1619 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5245.8 | uss_MB: 4860.8 | child_rss_MB: 7775.8 | allocated_MB: 395.0 | reserved_MB: 11792.0 | inactive_split_MB: 111.9 +rank1 [2026-10-27 20:12:14,504] INFO : Running validation on rank1 +rank1 [2026-10-27 20:12:14,504] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 20:14:09,431] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 20:14:09,866] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 20:14:34,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 20:14:35,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 20:14:35,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 20:14:36,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 20:14:36,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 20:14:36,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 20:14:37,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 20:14:37,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 20:14:47,790] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 20:14:47,943] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 20:14:48,315] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 20:15:03,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 20:15:03,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 20:15:04,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 20:15:04,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 20:15:05,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 20:15:05,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 20:15:06,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 20:15:06,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 20:15:16,804] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 20:15:17,038] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 20:15:17,439] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 20:15:32,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 20:15:33,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 20:15:33,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 20:15:34,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 20:15:34,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 20:15:34,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 20:15:35,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 20:15:35,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 20:15:45,869] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 20:19:21,751] INFO : Step 71000/100000 rank1 | Train Loss: 10.9922 | Train Loss Unweighted: 3.6772 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9201 | Regression_cos_phi: 0.0477 | Regression_energy: 0.2906 | Regression_eta: 0.1066 | Regression_pt: 0.2485 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6484 | physical_nopu: 4.4561 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3603 | Classification_binary: 2.9479 | Regression_cos_phi: 1.7678 | Regression_energy: 1.5610 | Regression_eta: 1.1714 | Regression_pt: 1.9828 | Regression_sin_phi: 1.6414 | LR: 1.10e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.5361s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0652 | max_forward: 0.2596 | max_loss: 0.0036 | max_backward: 0.3497 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 1.8772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 7279.8 | uss_MB: 6900.2 | child_rss_MB: 7804.4 | allocated_MB: 405.9 | reserved_MB: 11130.0 | inactive_split_MB: 135.5 +rank1 [2026-10-27 20:22:34,777] INFO : Step 72000/100000 rank1 | Train Loss: 14.4733 | Train Loss Unweighted: 4.6898 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3667 | Regression_cos_phi: 0.0458 | Regression_energy: 0.5948 | Regression_eta: 0.0650 | Regression_pt: 0.5567 | Regression_sin_phi: 0.0444 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4536 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2854 | Classification_binary: 3.4134 | Regression_cos_phi: 1.8786 | Regression_energy: 1.7709 | Regression_eta: 0.9134 | Regression_pt: 2.2119 | Regression_sin_phi: 1.8242 | LR: 1.05e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.5071s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1390 | max_loss: 0.0037 | max_backward: 0.1007 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.2476 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1281 | Memory: rss_MB: 5316.2 | uss_MB: 4936.1 | child_rss_MB: 7727.0 | allocated_MB: 393.8 | reserved_MB: 11706.0 | inactive_split_MB: 88.5 +rank1 [2026-10-27 20:25:34,428] INFO : Step 73000/100000 rank1 | Train Loss: 11.7903 | Train Loss Unweighted: 4.2561 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.5010 | Regression_cos_phi: 0.0578 | Regression_energy: 0.3092 | Regression_eta: 0.0644 | Regression_pt: 0.2565 | Regression_sin_phi: 0.0557 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4567 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3098 | Classification_binary: 3.4790 | Regression_cos_phi: 1.9984 | Regression_energy: 1.5667 | Regression_eta: 0.8327 | Regression_pt: 1.8100 | Regression_sin_phi: 1.8652 | LR: 1.01e-04 | DataLoad Time: 0.0844s | Model Forward Time: 2.5242s | Phase Times: max_pre_step_barrier: 0.1246 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0122 | max_backward: 0.1972 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.2348 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5307.2 | uss_MB: 4937.3 | child_rss_MB: 7690.0 | allocated_MB: 393.5 | reserved_MB: 11706.0 | inactive_split_MB: 93.2 +rank1 [2026-10-27 20:28:47,370] INFO : Step 74000/100000 rank1 | Train Loss: 11.6748 | Train Loss Unweighted: 4.2803 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4309 | Regression_cos_phi: 0.0428 | Regression_energy: 0.3572 | Regression_eta: 0.0695 | Regression_pt: 0.3177 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.6485 | physical_nopu: 4.4502 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3711 | Classification_binary: 3.6043 | Regression_cos_phi: 1.7325 | Regression_energy: 1.8785 | Regression_eta: 0.7628 | Regression_pt: 2.4568 | Regression_sin_phi: 1.3411 | LR: 9.68e-05 | DataLoad Time: 0.0014s | Model Forward Time: 3.0487s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1195 | max_loss: 0.0060 | max_backward: 0.2082 | max_optimizer_step: 0.1763 | max_tensorboard_logging: 1.8023 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5300.9 | uss_MB: 4938.3 | child_rss_MB: 7614.5 | allocated_MB: 394.3 | reserved_MB: 11706.0 | inactive_split_MB: 97.0 +rank1 [2026-10-27 20:31:56,443] INFO : Step 75000/100000 rank1 | Train Loss: 14.1604 | Train Loss Unweighted: 4.6445 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.4007 | Regression_cos_phi: 0.0489 | Regression_energy: 0.5577 | Regression_eta: 0.0640 | Regression_pt: 0.5112 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.6489 | physical_nopu: 4.4484 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3144 | Classification_binary: 3.4578 | Regression_cos_phi: 1.7687 | Regression_energy: 1.9528 | Regression_eta: 0.6893 | Regression_pt: 2.5613 | Regression_sin_phi: 1.7450 | LR: 9.27e-05 | DataLoad Time: 0.0026s | Model Forward Time: 2.9325s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0831 | max_loss: 0.0038 | max_backward: 0.2058 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.5751 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1279 | Memory: rss_MB: 5290.9 | uss_MB: 4938.0 | child_rss_MB: 7581.3 | allocated_MB: 394.4 | reserved_MB: 11706.0 | inactive_split_MB: 89.1 +rank1 [2026-10-27 20:34:57,734] INFO : Step 76000/100000 rank1 | Train Loss: 11.4149 | Train Loss Unweighted: 4.2984 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5224 | Regression_cos_phi: 0.0478 | Regression_energy: 0.3298 | Regression_eta: 0.0709 | Regression_pt: 0.2685 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6454 | physical_nopu: 4.4486 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2549 | Classification_binary: 3.4241 | Regression_cos_phi: 1.3000 | Regression_energy: 1.7410 | Regression_eta: 0.6330 | Regression_pt: 2.1771 | Regression_sin_phi: 1.4144 | LR: 8.88e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.1734s | Phase Times: max_pre_step_barrier: 0.0296 | max_batch_to_device: 0.0996 | max_forward: 0.1816 | max_loss: 0.0036 | max_backward: 0.0930 | max_optimizer_step: 0.2829 | max_tensorboard_logging: 1.7104 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1287 | Memory: rss_MB: 5286.0 | uss_MB: 4937.3 | child_rss_MB: 7586.8 | allocated_MB: 393.6 | reserved_MB: 12330.0 | inactive_split_MB: 97.7 +rank1 [2026-10-27 20:38:16,884] INFO : Step 77000/100000 rank1 | Train Loss: 12.8789 | Train Loss Unweighted: 4.4653 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4333 | Regression_cos_phi: 0.0478 | Regression_energy: 0.4517 | Regression_eta: 0.0624 | Regression_pt: 0.4079 | Regression_sin_phi: 0.0461 | Group Train Loss Unweighted (window mean): physical_pu: 3.6433 | physical_nopu: 4.4448 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4146 | Classification_binary: 3.4964 | Regression_cos_phi: 1.7739 | Regression_energy: 2.2654 | Regression_eta: 0.7655 | Regression_pt: 2.9737 | Regression_sin_phi: 1.3739 | LR: 8.50e-05 | DataLoad Time: 0.0026s | Model Forward Time: 3.0356s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.0831 | max_loss: 0.0514 | max_backward: 0.1524 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.8652 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5273.0 | uss_MB: 4939.8 | child_rss_MB: 7463.9 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 99.2 +rank1 [2026-10-27 20:41:21,660] INFO : Step 78000/100000 rank1 | Train Loss: 13.2420 | Train Loss Unweighted: 4.6345 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.5815 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4598 | Regression_eta: 0.0679 | Regression_pt: 0.4144 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6415 | physical_nopu: 4.4494 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3215 | Classification_binary: 3.5889 | Regression_cos_phi: 1.6519 | Regression_energy: 1.7157 | Regression_eta: 0.9813 | Regression_pt: 2.1138 | Regression_sin_phi: 2.1694 | LR: 8.13e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.0719s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0927 | max_loss: 0.0036 | max_backward: 0.2060 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.8450 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5261.0 | uss_MB: 4940.8 | child_rss_MB: 7383.4 | allocated_MB: 394.7 | reserved_MB: 12330.0 | inactive_split_MB: 95.1 +rank1 [2026-10-27 20:44:31,405] INFO : Step 79000/100000 rank1 | Train Loss: 11.5047 | Train Loss Unweighted: 4.2761 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4919 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3261 | Regression_eta: 0.0640 | Regression_pt: 0.2834 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6416 | physical_nopu: 4.4460 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3682 | Classification_binary: 3.3718 | Regression_cos_phi: 1.7094 | Regression_energy: 1.4826 | Regression_eta: 0.6793 | Regression_pt: 1.9773 | Regression_sin_phi: 1.0211 | LR: 7.78e-05 | DataLoad Time: 0.0026s | Model Forward Time: 3.2156s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0253 | max_forward: 0.1473 | max_loss: 0.0038 | max_backward: 0.0955 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 1.9569 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5248.8 | uss_MB: 4513.6 | child_rss_MB: 7321.5 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 120.0 +rank1 [2026-10-27 20:47:43,818] INFO : Step 80000/100000 rank1 | Train Loss: 15.3065 | Train Loss Unweighted: 4.8564 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4208 | Regression_cos_phi: 0.0455 | Regression_energy: 0.6415 | Regression_eta: 0.0773 | Regression_pt: 0.6092 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6398 | physical_nopu: 4.4406 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3513 | Classification_binary: 3.4935 | Regression_cos_phi: 1.6692 | Regression_energy: 2.5737 | Regression_eta: 0.9978 | Regression_pt: 3.6644 | Regression_sin_phi: 1.7326 | LR: 7.44e-05 | DataLoad Time: 0.0091s | Model Forward Time: 3.0632s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0010 | max_forward: 0.0840 | max_loss: 0.0039 | max_backward: 0.0916 | max_optimizer_step: 0.2630 | max_tensorboard_logging: 1.7339 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5238.8 | uss_MB: 4942.0 | child_rss_MB: 7231.1 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 141.7 +rank1 [2026-10-27 20:47:44,748] INFO : Running validation on rank1 +rank1 [2026-10-27 20:47:44,748] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 20:47:44,748] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 20:47:44,748] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 20:49:38,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 20:49:38,981] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 20:50:04,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 20:50:05,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 20:50:05,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 20:50:06,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 20:50:06,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 20:50:07,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 20:50:07,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 20:50:08,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 20:50:18,120] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 20:50:18,317] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 20:50:18,743] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 20:50:33,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 20:50:34,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 20:50:34,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 20:50:35,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 20:50:35,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 20:50:36,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 20:50:36,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 20:50:37,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 20:50:47,133] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 20:50:47,266] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 20:50:47,676] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 20:51:02,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 20:51:03,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 20:51:03,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 20:51:03,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 20:51:04,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 20:51:04,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 20:51:05,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 20:51:05,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 20:51:15,779] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 20:54:47,405] INFO : Step 81000/100000 rank1 | Train Loss: 10.7072 | Train Loss Unweighted: 3.6241 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.9080 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2710 | Regression_eta: 0.1078 | Regression_pt: 0.2269 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6413 | physical_nopu: 4.4395 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3290 | Classification_binary: 2.8937 | Regression_cos_phi: 1.6521 | Regression_energy: 1.3063 | Regression_eta: 1.1468 | Regression_pt: 1.5654 | Regression_sin_phi: 1.5806 | LR: 7.11e-05 | DataLoad Time: 0.0014s | Model Forward Time: 3.4730s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0033 | max_forward: 0.1984 | max_loss: 0.0038 | max_backward: 0.3244 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 1.8991 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 7314.5 | uss_MB: 7027.3 | child_rss_MB: 7215.1 | allocated_MB: 408.0 | reserved_MB: 11650.0 | inactive_split_MB: 261.6 +rank1 [2026-10-27 20:58:04,709] INFO : Step 82000/100000 rank1 | Train Loss: 12.9094 | Train Loss Unweighted: 4.6043 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5247 | Regression_cos_phi: 0.0417 | Regression_energy: 0.4765 | Regression_eta: 0.0603 | Regression_pt: 0.4449 | Regression_sin_phi: 0.0402 | Group Train Loss Unweighted (window mean): physical_pu: 3.6409 | physical_nopu: 4.4420 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3306 | Classification_binary: 3.6201 | Regression_cos_phi: 1.4449 | Regression_energy: 2.6335 | Regression_eta: 0.7238 | Regression_pt: 3.7101 | Regression_sin_phi: 1.5279 | LR: 6.80e-05 | DataLoad Time: 0.0011s | Model Forward Time: 3.1718s | Phase Times: max_pre_step_barrier: 0.0156 | max_batch_to_device: 0.0008 | max_forward: 0.1472 | max_loss: 0.0038 | max_backward: 0.1553 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 1.8498 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5342.3 | uss_MB: 5059.9 | child_rss_MB: 7159.0 | allocated_MB: 393.8 | reserved_MB: 11650.0 | inactive_split_MB: 92.9 +rank1 [2026-10-27 21:01:20,554] INFO : Step 83000/100000 rank1 | Train Loss: 11.2556 | Train Loss Unweighted: 4.2618 | Train Loss Components: Classification: 0.0094 | Classification_binary: 3.5167 | Regression_cos_phi: 0.0501 | Regression_energy: 0.3127 | Regression_eta: 0.0669 | Regression_pt: 0.2586 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6408 | physical_nopu: 4.4397 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1649 | Classification_binary: 3.3480 | Regression_cos_phi: 1.6067 | Regression_energy: 1.3156 | Regression_eta: 0.5670 | Regression_pt: 1.5233 | Regression_sin_phi: 1.3467 | LR: 6.51e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.5750s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0008 | max_forward: 0.1683 | max_loss: 0.0037 | max_backward: 0.1023 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.3519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 5336.3 | uss_MB: 5061.6 | child_rss_MB: 7095.7 | allocated_MB: 393.2 | reserved_MB: 11650.0 | inactive_split_MB: 95.2 +rank1 [2026-10-27 21:04:33,664] INFO : Step 84000/100000 rank1 | Train Loss: 11.5964 | Train Loss Unweighted: 4.2611 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4346 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3486 | Regression_eta: 0.0638 | Regression_pt: 0.3055 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6366 | physical_nopu: 4.4419 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3553 | Classification_binary: 3.3556 | Regression_cos_phi: 1.5454 | Regression_energy: 1.9191 | Regression_eta: 0.5284 | Regression_pt: 2.4743 | Regression_sin_phi: 1.4803 | LR: 6.23e-05 | DataLoad Time: 0.0014s | Model Forward Time: 4.9343s | Phase Times: max_pre_step_barrier: 0.2933 | max_batch_to_device: 0.0008 | max_forward: 0.1129 | max_loss: 0.0110 | max_backward: 0.1312 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 3.4426 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5330.3 | uss_MB: 5063.1 | child_rss_MB: 7029.9 | allocated_MB: 393.9 | reserved_MB: 11650.0 | inactive_split_MB: 95.6 +rank1 [2026-10-27 21:07:28,408] INFO : Step 85000/100000 rank1 | Train Loss: 14.1090 | Train Loss Unweighted: 4.6641 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.3775 | Regression_cos_phi: 0.0425 | Regression_energy: 0.5754 | Regression_eta: 0.0615 | Regression_pt: 0.5479 | Regression_sin_phi: 0.0420 | Group Train Loss Unweighted (window mean): physical_pu: 3.6325 | physical_nopu: 4.4336 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4293 | Classification_binary: 3.5654 | Regression_cos_phi: 1.8524 | Regression_energy: 2.1213 | Regression_eta: 0.6369 | Regression_pt: 2.7639 | Regression_sin_phi: 1.6683 | LR: 5.96e-05 | DataLoad Time: 0.0018s | Model Forward Time: 2.5394s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0496 | max_backward: 0.1488 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 1.3583 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 5323.1 | uss_MB: 5063.4 | child_rss_MB: 7037.3 | allocated_MB: 394.0 | reserved_MB: 11698.0 | inactive_split_MB: 93.1 +rank1 [2026-10-27 21:10:26,799] INFO : Step 86000/100000 rank1 | Train Loss: 11.7921 | Train Loss Unweighted: 4.3412 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.5823 | Regression_cos_phi: 0.0537 | Regression_energy: 0.3118 | Regression_eta: 0.0750 | Regression_pt: 0.2521 | Regression_sin_phi: 0.0550 | Group Train Loss Unweighted (window mean): physical_pu: 3.6404 | physical_nopu: 4.4337 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2593 | Classification_binary: 3.5455 | Regression_cos_phi: 1.8733 | Regression_energy: 1.5213 | Regression_eta: 0.7990 | Regression_pt: 1.8902 | Regression_sin_phi: 1.7082 | LR: 5.71e-05 | DataLoad Time: 0.0020s | Model Forward Time: 2.4341s | Phase Times: max_pre_step_barrier: 0.0233 | max_batch_to_device: 0.0898 | max_forward: 0.1824 | max_loss: 0.0038 | max_backward: 0.0938 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.9995 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1323 | Memory: rss_MB: 5319.1 | uss_MB: 4616.7 | child_rss_MB: 7006.7 | allocated_MB: 393.2 | reserved_MB: 11704.0 | inactive_split_MB: 96.0 +rank1 [2026-10-27 21:13:31,178] INFO : Step 87000/100000 rank1 | Train Loss: 12.0054 | Train Loss Unweighted: 4.3232 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.3964 | Regression_cos_phi: 0.0444 | Regression_energy: 0.4046 | Regression_eta: 0.0620 | Regression_pt: 0.3567 | Regression_sin_phi: 0.0418 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3900 | Classification_binary: 3.5930 | Regression_cos_phi: 2.0099 | Regression_energy: 2.1734 | Regression_eta: 0.6623 | Regression_pt: 2.8355 | Regression_sin_phi: 1.6775 | LR: 5.48e-05 | DataLoad Time: 0.0008s | Model Forward Time: 2.3335s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.1290 | max_loss: 0.0039 | max_backward: 0.0975 | max_optimizer_step: 0.1784 | max_tensorboard_logging: 1.1620 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5310.9 | uss_MB: 5064.7 | child_rss_MB: 7011.5 | allocated_MB: 394.7 | reserved_MB: 11704.0 | inactive_split_MB: 94.3 +rank1 [2026-10-27 21:16:33,434] INFO : Step 88000/100000 rank1 | Train Loss: 14.2663 | Train Loss Unweighted: 4.6673 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.4533 | Regression_cos_phi: 0.0523 | Regression_energy: 0.5342 | Regression_eta: 0.0666 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0508 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3333 | Classification_binary: 3.2511 | Regression_cos_phi: 1.4396 | Regression_energy: 3.0852 | Regression_eta: 0.6356 | Regression_pt: 4.2800 | Regression_sin_phi: 1.4092 | LR: 5.26e-05 | DataLoad Time: 0.0021s | Model Forward Time: 2.3220s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0008 | max_forward: 0.0997 | max_loss: 0.0039 | max_backward: 0.1038 | max_optimizer_step: 0.2517 | max_tensorboard_logging: 1.0175 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5309.9 | uss_MB: 5065.2 | child_rss_MB: 7012.4 | allocated_MB: 393.7 | reserved_MB: 11704.0 | inactive_split_MB: 103.2 +rank1 [2026-10-27 21:19:30,808] INFO : Step 89000/100000 rank1 | Train Loss: 11.9885 | Train Loss Unweighted: 4.3122 | Train Loss Components: Classification: 0.0138 | Classification_binary: 3.4837 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3438 | Regression_eta: 0.0687 | Regression_pt: 0.2984 | Regression_sin_phi: 0.0554 | Group Train Loss Unweighted (window mean): physical_pu: 3.6329 | physical_nopu: 4.4374 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3705 | Classification_binary: 3.5533 | Regression_cos_phi: 2.2000 | Regression_energy: 1.9592 | Regression_eta: 0.7229 | Regression_pt: 2.4419 | Regression_sin_phi: 2.4719 | LR: 5.06e-05 | DataLoad Time: 0.0018s | Model Forward Time: 2.2338s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0976 | max_loss: 0.0053 | max_backward: 0.1242 | max_optimizer_step: 0.1785 | max_tensorboard_logging: 1.0326 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 5307.5 | uss_MB: 5066.4 | child_rss_MB: 7006.7 | allocated_MB: 395.0 | reserved_MB: 13422.0 | inactive_split_MB: 109.9 +rank1 [2026-10-27 21:22:22,522] INFO : Step 90000/100000 rank1 | Train Loss: 10.5888 | Train Loss Unweighted: 3.6256 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9282 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2656 | Regression_eta: 0.1044 | Regression_pt: 0.2178 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6354 | physical_nopu: 4.4354 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3357 | Classification_binary: 2.9578 | Regression_cos_phi: 1.6980 | Regression_energy: 1.2809 | Regression_eta: 1.1316 | Regression_pt: 1.5535 | Regression_sin_phi: 1.5597 | LR: 4.88e-05 | DataLoad Time: 0.0022s | Model Forward Time: 4.1184s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0033 | max_forward: 0.1953 | max_loss: 0.0645 | max_backward: 0.8258 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.9975 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5307.5 | uss_MB: 5074.7 | child_rss_MB: 7021.2 | allocated_MB: 404.6 | reserved_MB: 13422.0 | inactive_split_MB: 208.4 +rank1 [2026-10-27 21:22:23,503] INFO : Running validation on rank1 +rank1 [2026-10-27 21:22:23,503] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 21:22:23,504] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 21:22:23,504] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 21:22:23,504] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 21:24:23,625] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 21:24:24,116] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 21:24:48,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 21:24:49,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 21:24:49,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 21:24:50,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 21:24:50,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 21:24:51,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 21:24:51,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 21:24:52,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 21:25:02,153] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 21:25:02,472] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 21:25:02,869] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 21:25:18,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 21:25:18,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 21:25:19,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 21:25:19,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 21:25:19,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 21:25:20,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 21:25:20,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 21:25:21,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 21:25:31,370] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 21:25:32,094] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 21:25:32,491] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 21:25:47,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 21:25:48,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 21:25:48,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 21:25:49,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 21:25:49,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 21:25:50,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 21:25:50,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 21:25:51,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 21:26:01,273] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 21:29:48,334] INFO : Step 91000/100000 rank1 | Train Loss: 11.9808 | Train Loss Unweighted: 4.3520 | Train Loss Components: Classification: 0.0128 | Classification_binary: 3.5260 | Regression_cos_phi: 0.0522 | Regression_energy: 0.3480 | Regression_eta: 0.0723 | Regression_pt: 0.2901 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4346 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3253 | Classification_binary: 3.6445 | Regression_cos_phi: 2.3069 | Regression_energy: 1.7990 | Regression_eta: 1.3323 | Regression_pt: 2.1261 | Regression_sin_phi: 1.5159 | LR: 4.71e-05 | DataLoad Time: 0.0021s | Model Forward Time: 3.6013s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1743 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.3267 | max_tensorboard_logging: 2.0733 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0716 | Memory: rss_MB: 7308.2 | uss_MB: 7066.1 | child_rss_MB: 7044.9 | allocated_MB: 393.7 | reserved_MB: 11392.0 | inactive_split_MB: 89.1 +rank1 [2026-10-27 21:33:08,656] INFO : Step 92000/100000 rank1 | Train Loss: 14.4974 | Train Loss Unweighted: 4.7629 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4460 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5917 | Regression_eta: 0.0739 | Regression_pt: 0.5467 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6297 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4161 | Classification_binary: 3.3434 | Regression_cos_phi: 1.7259 | Regression_energy: 3.0452 | Regression_eta: 1.1659 | Regression_pt: 4.1881 | Regression_sin_phi: 1.1510 | LR: 4.57e-05 | DataLoad Time: 0.0011s | Model Forward Time: 2.6203s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1620 | max_loss: 0.0037 | max_backward: 0.2478 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 1.1676 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0595 | Memory: rss_MB: 7302.2 | uss_MB: 6481.6 | child_rss_MB: 6997.3 | allocated_MB: 394.0 | reserved_MB: 11392.0 | inactive_split_MB: 90.6 +rank1 [2026-10-27 21:36:11,295] INFO : Step 93000/100000 rank1 | Train Loss: 13.1041 | Train Loss Unweighted: 4.5189 | Train Loss Components: Classification: 0.0119 | Classification_binary: 3.5265 | Regression_cos_phi: 0.0575 | Regression_energy: 0.4342 | Regression_eta: 0.0649 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6346 | physical_nopu: 4.4327 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2162 | Classification_binary: 3.6152 | Regression_cos_phi: 2.0958 | Regression_energy: 1.8240 | Regression_eta: 0.6670 | Regression_pt: 2.2168 | Regression_sin_phi: 1.6852 | LR: 4.43e-05 | DataLoad Time: 0.0020s | Model Forward Time: 2.4221s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1013 | max_loss: 0.1069 | max_backward: 0.2035 | max_optimizer_step: 0.1792 | max_tensorboard_logging: 1.1596 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 5379.1 | uss_MB: 5141.2 | child_rss_MB: 6951.6 | allocated_MB: 393.6 | reserved_MB: 13294.0 | inactive_split_MB: 89.2 +rank1 [2026-10-27 21:39:11,302] INFO : Step 94000/100000 rank1 | Train Loss: 11.5177 | Train Loss Unweighted: 4.2756 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4441 | Regression_cos_phi: 0.0413 | Regression_energy: 0.3503 | Regression_eta: 0.0686 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6308 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3111 | Classification_binary: 3.3015 | Regression_cos_phi: 1.3934 | Regression_energy: 2.0411 | Regression_eta: 0.9218 | Regression_pt: 2.4449 | Regression_sin_phi: 1.4092 | LR: 4.32e-05 | DataLoad Time: 0.0023s | Model Forward Time: 6.3162s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0191 | max_forward: 0.1721 | max_loss: 0.0188 | max_backward: 0.2028 | max_optimizer_step: 0.2895 | max_tensorboard_logging: 4.7123 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 5372.1 | uss_MB: 5157.2 | child_rss_MB: 6925.7 | allocated_MB: 394.1 | reserved_MB: 13294.0 | inactive_split_MB: 89.1 +rank1 [2026-10-27 21:42:20,694] INFO : Step 95000/100000 rank1 | Train Loss: 14.1207 | Train Loss Unweighted: 4.6477 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3920 | Regression_cos_phi: 0.0441 | Regression_energy: 0.5776 | Regression_eta: 0.0602 | Regression_pt: 0.5087 | Regression_sin_phi: 0.0499 | Group Train Loss Unweighted (window mean): physical_pu: 3.6267 | physical_nopu: 4.4321 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4082 | Classification_binary: 3.0935 | Regression_cos_phi: 1.4895 | Regression_energy: 3.6007 | Regression_eta: 0.5441 | Regression_pt: 5.1083 | Regression_sin_phi: 1.0313 | LR: 4.22e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.5035s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.0941 | max_loss: 0.0036 | max_backward: 0.1734 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 1.2811 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5361.0 | uss_MB: 5157.8 | child_rss_MB: 6928.3 | allocated_MB: 396.0 | reserved_MB: 14294.0 | inactive_split_MB: 91.1 +rank1 [2026-10-27 21:45:16,250] INFO : Step 96000/100000 rank1 | Train Loss: 11.5036 | Train Loss Unweighted: 4.2276 | Train Loss Components: Classification: 0.0097 | Classification_binary: 3.4897 | Regression_cos_phi: 0.0538 | Regression_energy: 0.2982 | Regression_eta: 0.0802 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6278 | physical_nopu: 4.4329 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2198 | Classification_binary: 3.5536 | Regression_cos_phi: 1.7669 | Regression_energy: 1.5627 | Regression_eta: 1.1606 | Regression_pt: 1.9841 | Regression_sin_phi: 2.0001 | LR: 4.14e-05 | DataLoad Time: 0.0022s | Model Forward Time: 2.3868s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0007 | max_forward: 0.0991 | max_loss: 0.0037 | max_backward: 0.1000 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5357.0 | uss_MB: 5158.4 | child_rss_MB: 6899.7 | allocated_MB: 393.0 | reserved_MB: 14294.0 | inactive_split_MB: 95.3 +rank1 [2026-10-27 21:48:34,573] INFO : Step 97000/100000 rank1 | Train Loss: 11.8793 | Train Loss Unweighted: 4.3044 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.4420 | Regression_cos_phi: 0.0454 | Regression_energy: 0.3663 | Regression_eta: 0.0711 | Regression_pt: 0.3156 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.6277 | physical_nopu: 4.4255 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3827 | Classification_binary: 3.6312 | Regression_cos_phi: 1.5270 | Regression_energy: 1.7860 | Regression_eta: 0.7972 | Regression_pt: 2.2136 | Regression_sin_phi: 1.4708 | LR: 4.08e-05 | DataLoad Time: 0.0022s | Model Forward Time: 2.5910s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1192 | max_loss: 0.0048 | max_backward: 0.1852 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 5356.0 | uss_MB: 5157.9 | child_rss_MB: 6903.2 | allocated_MB: 394.7 | reserved_MB: 14294.0 | inactive_split_MB: 98.1 +rank1 [2026-10-27 21:51:32,851] INFO : Step 98000/100000 rank1 | Train Loss: 12.9367 | Train Loss Unweighted: 4.4840 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0441 | Regression_energy: 0.4811 | Regression_eta: 0.0611 | Regression_pt: 0.4378 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3193 | Classification_binary: 3.3242 | Regression_cos_phi: 1.2542 | Regression_energy: 1.9518 | Regression_eta: 0.5520 | Regression_pt: 2.7485 | Regression_sin_phi: 1.1889 | LR: 4.04e-05 | DataLoad Time: 0.0016s | Model Forward Time: 2.8946s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1318 | max_loss: 0.0038 | max_backward: 0.1127 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.6868 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 5362.0 | uss_MB: 5160.3 | child_rss_MB: 6899.7 | allocated_MB: 394.5 | reserved_MB: 14294.0 | inactive_split_MB: 82.0 +rank1 [2026-10-27 21:54:35,739] INFO : Step 99000/100000 rank1 | Train Loss: 11.6408 | Train Loss Unweighted: 4.3204 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.5557 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3117 | Regression_eta: 0.0641 | Regression_pt: 0.2727 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.6292 | physical_nopu: 4.4284 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2062 | Classification_binary: 3.6693 | Regression_cos_phi: 1.8714 | Regression_energy: 1.3789 | Regression_eta: 0.8317 | Regression_pt: 1.6872 | Regression_sin_phi: 1.8076 | LR: 4.01e-05 | DataLoad Time: 0.0077s | Model Forward Time: 3.0811s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1731 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.8331 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 5358.0 | uss_MB: 5160.5 | child_rss_MB: 6886.0 | allocated_MB: 394.3 | reserved_MB: 14294.0 | inactive_split_MB: 113.7 +rank1 [2026-10-27 21:57:56,512] INFO : Step 100000/100000 rank1 | Train Loss: 10.8909 | Train Loss Unweighted: 3.6545 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9133 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2833 | Regression_eta: 0.1041 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6336 | physical_nopu: 4.4311 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3388 | Classification_binary: 2.9150 | Regression_cos_phi: 1.7014 | Regression_energy: 1.3178 | Regression_eta: 1.0630 | Regression_pt: 1.6214 | Regression_sin_phi: 1.5956 | LR: 4.00e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.7489s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0030 | max_forward: 0.2088 | max_loss: 0.0393 | max_backward: 0.3031 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.9276 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 5350.9 | uss_MB: 5163.3 | child_rss_MB: 6834.2 | allocated_MB: 405.7 | reserved_MB: 14294.0 | inactive_split_MB: 165.9 +rank1 [2026-10-27 21:57:57,456] INFO : Running validation on rank1 +rank1 [2026-10-27 21:57:57,456] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 21:57:57,456] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 21:57:57,456] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 21:57:57,456] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 21:59:51,633] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 21:59:52,049] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 22:00:18,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 22:00:19,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 22:00:19,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 22:00:20,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 22:00:20,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 22:00:21,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 22:00:21,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 22:00:21,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 22:00:31,957] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 22:00:32,061] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 22:00:32,464] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 22:00:47,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 22:00:48,240] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 22:00:48,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 22:00:49,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 22:00:49,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 22:00:50,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 22:00:50,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 22:00:51,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 22:01:01,064] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 22:01:01,522] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 22:01:01,928] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 22:01:16,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 22:01:17,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 22:01:17,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 22:01:18,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 22:01:18,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 22:01:19,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 22:01:19,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 22:01:20,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 22:01:30,107] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 22:01:31,009] INFO : Training completed. Total time on device 1: 350.241min diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.2 b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.2 new file mode 100644 index 0000000000000000000000000000000000000000..f817089a50569015c343273ed4b9034be2b8dcd4 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.2 @@ -0,0 +1,851 @@ +rank2 [2026-10-27 16:10:10,438] INFO : Initializing model with seed=12345; process seed=12347 +rank2 [2026-10-27 16:10:10,439] INFO : configured dtype=torch.bfloat16 for autocast +rank2 [2026-10-27 16:10:37,169] INFO : Instantiating model +rank2 [2026-10-27 16:10:37,169] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank2 [2026-10-27 16:10:37,169] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank2 [2026-10-27 16:10:37,201] INFO : Input encoding initialization took 0.03s +rank2 [2026-10-27 16:10:37,201] INFO : nn0 parameters: 2130944 +rank2 [2026-10-27 16:10:37,201] INFO : Initializing shared backbone layers, num_convs=3 +rank2 [2026-10-27 16:10:37,201] INFO : Initializing backbone layer 0 +rank2 [2026-10-27 16:10:37,201] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-27 16:10:37,212] INFO : Initializing backbone layer 1 +rank2 [2026-10-27 16:10:37,212] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-27 16:10:37,223] INFO : Initializing backbone layer 2 +rank2 [2026-10-27 16:10:37,223] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-27 16:10:37,233] INFO : Backbone initialization took 0.03s +rank2 [2026-10-27 16:10:37,234] INFO : backbone parameters: 4733952 +rank2 [2026-10-27 16:10:37,234] INFO : Initializing output DNNs +rank2 [2026-10-27 16:10:37,268] INFO : Output DNNs initialization took 0.03s +rank2 [2026-10-27 16:10:37,268] INFO : backbone_mode=BackboneMode.SHARED +rank2 [2026-10-27 16:10:37,268] INFO : jagged_attention enabled=True +rank2 [2026-10-27 16:10:37,268] INFO : flash_attn_varlen enabled=True +rank2 [2026-10-27 16:10:37,268] INFO : task_queries enabled=False +rank2 [2026-10-27 16:10:37,268] INFO : classification_norm parameters: 1024 +rank2 [2026-10-27 16:10:37,268] INFO : regression_norm parameters: 1024 +rank2 [2026-10-27 16:10:37,268] INFO : classification_query parameters: 0 +rank2 [2026-10-27 16:10:37,268] INFO : regression_query parameters: 0 +rank2 [2026-10-27 16:10:37,269] INFO : classification_readout parameters: 0 +rank2 [2026-10-27 16:10:37,269] INFO : regression_readout parameters: 0 +rank2 [2026-10-27 16:10:37,269] INFO : output_mode=elementwise +rank2 [2026-10-27 16:10:37,269] INFO : set_decoder parameters: 0 +rank2 [2026-10-27 16:10:37,269] INFO : nn_binary_particle parameters: 264706 +rank2 [2026-10-27 16:10:37,269] INFO : nn_pid parameters: 268297 +rank2 [2026-10-27 16:10:37,269] INFO : nn_pt parameters: 2113544 +rank2 [2026-10-27 16:10:37,269] INFO : nn_eta parameters: 264706 +rank2 [2026-10-27 16:10:37,269] INFO : nn_sin_phi parameters: 264706 +rank2 [2026-10-27 16:10:37,269] INFO : nn_cos_phi parameters: 264706 +rank2 [2026-10-27 16:10:37,269] INFO : nn_energy parameters: 2113544 +rank2 [2026-10-27 16:10:37,270] INFO : Total MLPF parameters: 12421153 +rank2 [2026-10-27 16:10:37,270] INFO : MLPF __init__ done +rank2 [2026-10-27 16:10:37,270] INFO : Instantiated model +rank2 [2026-10-27 16:10:37,270] INFO : Moving model to device rank=2 +rank2 [2026-10-27 16:10:37,299] INFO : Moved model to device rank=2 +rank2 [2026-10-27 16:10:37,301] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank2 [2026-10-27 16:10:37,301] INFO : Configured model for SyncBatchNorm rank=2 +rank2 [2026-10-27 16:10:37,493] INFO : Configured model for DistributedDataParallel rank=2 +rank2 [2026-10-27 16:10:37,594] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +167 module.nn_energy.nn.7.0.bias ... 0 +168 module.nn_energy.nn.7.2.weight ... 0 +169 module.nn_energy.nn.7.2.bias ... 0 +170 module.nn_energy.nn.7.4.weight ... 0 +171 module.nn_energy.nn.7.4.bias ... 0 + +[172 rows x 3 columns] +rank2 [2026-10-27 16:10:37,594] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=512, out_features=512, bias=True) + ) + (norm0): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=512, out_features=512, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank2 [2026-10-27 16:10:37,595] INFO : Trainable parameters: 12421153 +rank2 [2026-10-27 16:10:37,595] INFO : Non-trainable parameters: 0 +rank2 [2026-10-27 16:10:37,595] INFO : Total parameters: 12421153 +rank2 [2026-10-27 16:10:37,597] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.mha.in_proj_weight 786432 0 + module.backbone.0.mha.in_proj_bias 1536 0 +module.backbone.0.mha.out_proj.weight 262144 0 + module.backbone.0.mha.out_proj.bias 512 0 + module.backbone.0.norm0.weight 512 0 + module.backbone.0.norm0.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm1.bias 512 0 + module.backbone.0.seq.0.weight 262144 0 + module.backbone.0.seq.0.bias 512 0 + module.backbone.0.seq.2.weight 262144 0 + module.backbone.0.seq.2.bias 512 0 + module.backbone.1.mha.in_proj_weight 786432 0 + module.backbone.1.mha.in_proj_bias 1536 0 +module.backbone.1.mha.out_proj.weight 262144 0 + module.backbone.1.mha.out_proj.bias 512 0 + module.backbone.1.norm0.weight 512 0 + module.backbone.1.norm0.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm1.bias 512 0 + module.backbone.1.seq.0.weight 262144 0 + module.backbone.1.seq.0.bias 512 0 + module.backbone.1.seq.2.weight 262144 0 + module.backbone.1.seq.2.bias 512 0 + module.backbone.2.mha.in_proj_weight 786432 0 + module.backbone.2.mha.in_proj_bias 1536 0 +module.backbone.2.mha.out_proj.weight 262144 0 + module.backbone.2.mha.out_proj.bias 512 0 + module.backbone.2.norm0.weight 512 0 + module.backbone.2.norm0.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm1.bias 512 0 + module.backbone.2.seq.0.weight 262144 0 + module.backbone.2.seq.0.bias 512 0 + module.backbone.2.seq.2.weight 262144 0 + module.backbone.2.seq.2.bias 512 0 + module.classification_norm.weight 512 0 + module.classification_norm.bias 512 0 + module.regression_norm.weight 512 0 + module.regression_norm.bias 512 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank2 [2026-10-27 16:10:37,597] INFO : Getting dataloaders +rank2 [2026-10-27 16:10:37,597] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:43,313] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:44,186] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:44,826] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank2 [2026-10-27 16:10:44,826] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:46,052] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:46,992] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:47,782] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank2 [2026-10-27 16:10:47,782] INFO : Creating WeightedMixIterator with 2 data loaders. +rank2 [2026-10-27 16:10:47,783] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:47,935] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:48,086] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:48,237] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank2 [2026-10-27 16:10:48,238] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:48,401] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:48,567] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:48,726] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank2 [2026-10-27 16:10:48,726] INFO : Creating InterleavedIterator with 2 data loaders. +rank2 [2026-10-27 16:11:16,324] INFO : Got dataloaders +rank2 [2026-10-27 16:11:16,324] INFO : loader split=train rank=2 len=232058 +rank2 [2026-10-27 16:11:16,325] INFO : loader split=valid rank=2 len=48 +rank2 [2026-10-27 16:11:16,325] INFO : Starting training from step 1 to 100000 on rank 2 of 8 +rank2 [2026-10-27 16:11:16,325] INFO : Creating train_iterator +rank2 [2026-10-27 16:11:16,325] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank2 [2026-10-27 16:11:30,487] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank2 [2026-10-27 16:11:45,655] INFO : Created train_iterator +rank2 [2026-10-27 16:12:23,651] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.123548031449318, 'Classification': 0.2298217851272784, 'Regression_pt': 0.7194055180624127, 'Regression_eta': 0.4619348940253258, 'Regression_sin_phi': 0.15005176125559955, 'Regression_cos_phi': 0.14165307599119842, 'Regression_energy': 1.0569261094555258}, weights={'Classification_binary': 1.0, 'Classification': 22.293569564819336, 'Regression_pt': 7.12191915512085, 'Regression_eta': 11.091493606567383, 'Regression_sin_phi': 34.14520263671875, 'Regression_cos_phi': 36.16969299316406, 'Regression_energy': 4.847593307495117} +rank2 [2026-10-27 16:14:48,547] INFO : Step 1000/100000 rank2 | Train Loss: 15.9738 | Train Loss Unweighted: 5.5021 | Train Loss Components: Classification: 0.0191 | Classification_binary: 4.1436 | Regression_cos_phi: 0.0516 | Regression_energy: 0.6035 | Regression_eta: 0.0819 | Regression_pt: 0.5500 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 5.2846 | physical_nopu: 6.1663 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4949 | Classification_binary: 4.1021 | Regression_cos_phi: 1.6377 | Regression_energy: 3.2167 | Regression_eta: 0.8658 | Regression_pt: 4.4166 | Regression_sin_phi: 1.6606 | LR: 4.00e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.9582s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0013 | max_forward: 0.0194 | max_loss: 0.0039 | max_backward: 0.0308 | max_optimizer_step: 0.1391 | max_tensorboard_logging: 1.0816 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0030 | Memory: rss_MB: 4094.8 | uss_MB: 2885.6 | child_rss_MB: 10815.8 | allocated_MB: 395.5 | reserved_MB: 11198.0 | inactive_split_MB: 104.9 +rank2 [2026-10-27 16:17:31,218] INFO : Step 2000/100000 rank2 | Train Loss: 17.2752 | Train Loss Unweighted: 5.6831 | Train Loss Components: Classification: 0.0157 | Classification_binary: 4.0887 | Regression_cos_phi: 0.0506 | Regression_energy: 0.7236 | Regression_eta: 0.0696 | Regression_pt: 0.6798 | Regression_sin_phi: 0.0552 | Group Train Loss Unweighted (window mean): physical_pu: 4.4021 | physical_nopu: 5.4797 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2936 | Classification_binary: 4.2442 | Regression_cos_phi: 2.2809 | Regression_energy: 2.1081 | Regression_eta: 0.8506 | Regression_pt: 2.7890 | Regression_sin_phi: 1.6395 | LR: 4.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7792s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0250 | max_loss: 0.0036 | max_backward: 0.0657 | max_optimizer_step: 0.1397 | max_tensorboard_logging: 0.8949 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4097.7 | uss_MB: 2895.6 | child_rss_MB: 10834.7 | allocated_MB: 394.2 | reserved_MB: 11198.0 | inactive_split_MB: 90.1 +rank2 [2026-10-27 16:20:19,985] INFO : Step 3000/100000 rank2 | Train Loss: 14.0295 | Train Loss Unweighted: 5.1258 | Train Loss Components: Classification: 0.0148 | Classification_binary: 4.0405 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4749 | Regression_eta: 0.0746 | Regression_pt: 0.4205 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 4.1913 | physical_nopu: 5.3261 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3364 | Classification_binary: 4.1237 | Regression_cos_phi: 1.8914 | Regression_energy: 2.5857 | Regression_eta: 0.8963 | Regression_pt: 3.2125 | Regression_sin_phi: 1.9698 | LR: 3.99e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.4927s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1067 | max_loss: 0.0036 | max_backward: 0.0365 | max_optimizer_step: 0.1415 | max_tensorboard_logging: 1.1098 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0093 | Memory: rss_MB: 4102.6 | uss_MB: 2907.4 | child_rss_MB: 10837.4 | allocated_MB: 395.4 | reserved_MB: 11784.0 | inactive_split_MB: 88.5 +rank2 [2026-10-27 16:23:08,050] INFO : Step 4000/100000 rank2 | Train Loss: 11.7244 | Train Loss Unweighted: 3.9925 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.1718 | Regression_cos_phi: 0.0487 | Regression_energy: 0.3212 | Regression_eta: 0.1125 | Regression_pt: 0.2729 | Regression_sin_phi: 0.0492 | Group Train Loss Unweighted (window mean): physical_pu: 4.0838 | physical_nopu: 5.2370 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3734 | Classification_binary: 3.1775 | Regression_cos_phi: 1.7204 | Regression_energy: 1.5664 | Regression_eta: 1.1584 | Regression_pt: 1.9396 | Regression_sin_phi: 1.6716 | LR: 3.99e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.2689s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0037 | max_forward: 0.0726 | max_loss: 0.0066 | max_backward: 0.1670 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4108.6 | uss_MB: 2925.8 | child_rss_MB: 10838.2 | allocated_MB: 406.4 | reserved_MB: 11904.0 | inactive_split_MB: 179.6 +rank2 [2026-10-27 16:25:55,983] INFO : Step 5000/100000 rank2 | Train Loss: 13.1313 | Train Loss Unweighted: 4.9481 | Train Loss Components: Classification: 0.0138 | Classification_binary: 4.0144 | Regression_cos_phi: 0.0518 | Regression_energy: 0.3902 | Regression_eta: 0.0928 | Regression_pt: 0.3380 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 4.0057 | physical_nopu: 5.1582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2810 | Classification_binary: 3.9865 | Regression_cos_phi: 1.6263 | Regression_energy: 1.8099 | Regression_eta: 0.7815 | Regression_pt: 2.3085 | Regression_sin_phi: 1.5939 | LR: 3.98e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.3569s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0196 | max_loss: 0.0037 | max_backward: 0.0324 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 1.3896 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0068 | Memory: rss_MB: 4091.1 | uss_MB: 2927.1 | child_rss_MB: 10673.2 | allocated_MB: 393.9 | reserved_MB: 11904.0 | inactive_split_MB: 90.0 +rank2 [2026-10-27 16:28:58,362] INFO : Step 6000/100000 rank2 | Train Loss: 14.5226 | Train Loss Unweighted: 5.0639 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.8601 | Regression_cos_phi: 0.0525 | Regression_energy: 0.5359 | Regression_eta: 0.0694 | Regression_pt: 0.4825 | Regression_sin_phi: 0.0459 | Group Train Loss Unweighted (window mean): physical_pu: 3.9666 | physical_nopu: 5.0829 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3829 | Classification_binary: 3.7530 | Regression_cos_phi: 1.5275 | Regression_energy: 3.4398 | Regression_eta: 0.5610 | Regression_pt: 4.6425 | Regression_sin_phi: 1.2313 | LR: 3.97e-04 | DataLoad Time: 0.0273s | Model Forward Time: 3.5739s | Phase Times: max_pre_step_barrier: 0.0280 | max_batch_to_device: 0.0008 | max_forward: 0.0311 | max_loss: 0.0037 | max_backward: 0.0629 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 2.6604 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0119 | Memory: rss_MB: 4078.1 | uss_MB: 2934.4 | child_rss_MB: 10495.3 | allocated_MB: 394.6 | reserved_MB: 12524.0 | inactive_split_MB: 109.0 +rank2 [2026-10-27 16:31:51,869] INFO : Step 7000/100000 rank2 | Train Loss: 13.9772 | Train Loss Unweighted: 5.0398 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.9636 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4671 | Regression_eta: 0.0797 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0537 | Group Train Loss Unweighted (window mean): physical_pu: 3.9390 | physical_nopu: 5.0316 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2691 | Classification_binary: 4.0856 | Regression_cos_phi: 2.1983 | Regression_energy: 1.5391 | Regression_eta: 0.8035 | Regression_pt: 1.9251 | Regression_sin_phi: 1.9045 | LR: 3.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.1736s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0243 | max_loss: 0.0036 | max_backward: 0.0295 | max_optimizer_step: 0.2429 | max_tensorboard_logging: 2.1406 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 4070.6 | uss_MB: 2937.8 | child_rss_MB: 10370.1 | allocated_MB: 393.9 | reserved_MB: 12524.0 | inactive_split_MB: 107.5 +rank2 [2026-10-27 16:35:03,086] INFO : Step 8000/100000 rank2 | Train Loss: 13.6161 | Train Loss Unweighted: 4.9650 | Train Loss Components: Classification: 0.0187 | Classification_binary: 3.9275 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4519 | Regression_eta: 0.0615 | Regression_pt: 0.4056 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.9031 | physical_nopu: 4.9878 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4338 | Classification_binary: 3.8813 | Regression_cos_phi: 2.0431 | Regression_energy: 2.1461 | Regression_eta: 0.5379 | Regression_pt: 3.0222 | Regression_sin_phi: 1.7758 | LR: 3.94e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.9573s | Phase Times: max_pre_step_barrier: 0.0692 | max_batch_to_device: 0.0893 | max_forward: 0.1169 | max_loss: 0.0037 | max_backward: 0.0488 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 1.9012 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4069.6 | uss_MB: 2945.4 | child_rss_MB: 10376.1 | allocated_MB: 393.8 | reserved_MB: 12524.0 | inactive_split_MB: 105.1 +rank2 [2026-10-27 16:38:07,477] INFO : Step 9000/100000 rank2 | Train Loss: 15.1606 | Train Loss Unweighted: 5.0855 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.7493 | Regression_cos_phi: 0.0474 | Regression_energy: 0.5909 | Regression_eta: 0.0749 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.8927 | physical_nopu: 4.9579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3766 | Classification_binary: 3.8651 | Regression_cos_phi: 1.6958 | Regression_energy: 3.8853 | Regression_eta: 1.2105 | Regression_pt: 5.7473 | Regression_sin_phi: 2.2794 | LR: 3.93e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.8503s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0007 | max_forward: 0.0386 | max_loss: 0.0041 | max_backward: 0.0582 | max_optimizer_step: 0.1496 | max_tensorboard_logging: 1.9225 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 4056.6 | uss_MB: 2948.9 | child_rss_MB: 10378.8 | allocated_MB: 395.8 | reserved_MB: 12524.0 | inactive_split_MB: 89.5 +rank2 [2026-10-27 16:41:18,312] INFO : Step 10000/100000 rank2 | Train Loss: 12.2525 | Train Loss Unweighted: 4.6725 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8667 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3374 | Regression_eta: 0.0750 | Regression_pt: 0.2772 | Regression_sin_phi: 0.0525 | Group Train Loss Unweighted (window mean): physical_pu: 3.8762 | physical_nopu: 4.9262 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2033 | Classification_binary: 3.7046 | Regression_cos_phi: 1.5439 | Regression_energy: 1.6693 | Regression_eta: 1.0218 | Regression_pt: 2.0723 | Regression_sin_phi: 1.4683 | LR: 3.91e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.1909s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0277 | max_loss: 0.0035 | max_backward: 0.0304 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 2.2107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4024.3 | uss_MB: 2953.8 | child_rss_MB: 10382.4 | allocated_MB: 393.6 | reserved_MB: 12524.0 | inactive_split_MB: 89.8 +rank2 [2026-10-27 16:41:19,251] INFO : Running validation on rank2 +rank2 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 16:43:24,651] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:43:25,081] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 16:43:52,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 16:43:53,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 16:43:53,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 16:43:54,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 16:43:54,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 16:43:55,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 16:43:55,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 16:43:56,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 16:44:06,238] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-27 16:44:06,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:44:06,938] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 16:44:21,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 16:44:22,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 16:44:22,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 16:44:23,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 16:44:23,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 16:44:24,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 16:44:24,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 16:44:24,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 16:44:35,009] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 16:44:35,783] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:44:36,273] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 16:44:51,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 16:44:51,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 16:44:52,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 16:44:52,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 16:44:52,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 16:44:53,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 16:44:53,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 16:44:54,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 16:45:04,270] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 16:48:51,455] INFO : Step 11000/100000 rank2 | Train Loss: 13.5331 | Train Loss Unweighted: 4.8490 | Train Loss Components: Classification: 0.0178 | Classification_binary: 3.8332 | Regression_cos_phi: 0.0514 | Regression_energy: 0.4326 | Regression_eta: 0.0715 | Regression_pt: 0.3906 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.8629 | physical_nopu: 4.8860 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4212 | Classification_binary: 3.7659 | Regression_cos_phi: 1.5481 | Regression_energy: 1.8291 | Regression_eta: 0.6535 | Regression_pt: 2.5446 | Regression_sin_phi: 1.6010 | LR: 3.89e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.1304s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0010 | max_forward: 0.0938 | max_loss: 0.0036 | max_backward: 0.1077 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.9405 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6748.2 | uss_MB: 5693.6 | child_rss_MB: 10378.2 | allocated_MB: 394.1 | reserved_MB: 11016.0 | inactive_split_MB: 102.9 +rank2 [2026-10-27 16:51:46,972] INFO : Step 12000/100000 rank2 | Train Loss: 16.0880 | Train Loss Unweighted: 5.2955 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.7102 | Regression_cos_phi: 0.0409 | Regression_energy: 0.7300 | Regression_eta: 0.0621 | Regression_pt: 0.6963 | Regression_sin_phi: 0.0392 | Group Train Loss Unweighted (window mean): physical_pu: 3.8522 | physical_nopu: 4.8676 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4120 | Classification_binary: 3.6517 | Regression_cos_phi: 1.2404 | Regression_energy: 3.9210 | Regression_eta: 0.5176 | Regression_pt: 5.5546 | Regression_sin_phi: 1.3558 | LR: 3.87e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.9531s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1050 | max_forward: 0.2073 | max_loss: 0.0038 | max_backward: 0.1040 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 1.6163 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6734.2 | uss_MB: 5695.7 | child_rss_MB: 10317.1 | allocated_MB: 394.0 | reserved_MB: 11384.0 | inactive_split_MB: 88.0 +rank2 [2026-10-27 16:54:47,130] INFO : Step 13000/100000 rank2 | Train Loss: 12.2526 | Train Loss Unweighted: 4.6994 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.8775 | Regression_cos_phi: 0.0480 | Regression_energy: 0.3426 | Regression_eta: 0.0688 | Regression_pt: 0.2970 | Regression_sin_phi: 0.0539 | Group Train Loss Unweighted (window mean): physical_pu: 3.8404 | physical_nopu: 4.8415 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2982 | Classification_binary: 4.1533 | Regression_cos_phi: 1.7610 | Regression_energy: 1.4674 | Regression_eta: 0.5157 | Regression_pt: 1.8218 | Regression_sin_phi: 1.7681 | LR: 3.85e-04 | DataLoad Time: 0.0053s | Model Forward Time: 3.3099s | Phase Times: max_pre_step_barrier: 0.0143 | max_batch_to_device: 0.0010 | max_forward: 0.0740 | max_loss: 0.0036 | max_backward: 0.0639 | max_optimizer_step: 0.2642 | max_tensorboard_logging: 2.1016 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 5345.9 | uss_MB: 4318.0 | child_rss_MB: 10308.5 | allocated_MB: 394.7 | reserved_MB: 12168.0 | inactive_split_MB: 111.5 +rank2 [2026-10-27 16:57:46,633] INFO : Step 14000/100000 rank2 | Train Loss: 11.1328 | Train Loss Unweighted: 3.8032 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.0595 | Regression_cos_phi: 0.0494 | Regression_energy: 0.2823 | Regression_eta: 0.1077 | Regression_pt: 0.2396 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.8281 | physical_nopu: 4.8178 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3656 | Classification_binary: 3.0188 | Regression_cos_phi: 1.7779 | Regression_energy: 1.3670 | Regression_eta: 1.1479 | Regression_pt: 1.7262 | Regression_sin_phi: 1.5815 | LR: 3.83e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.7656s | Phase Times: max_pre_step_barrier: 0.1064 | max_batch_to_device: 0.0033 | max_forward: 0.1130 | max_loss: 0.0036 | max_backward: 0.2788 | max_optimizer_step: 0.1663 | max_tensorboard_logging: 2.1758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1099 | Memory: rss_MB: 5335.9 | uss_MB: 4329.8 | child_rss_MB: 10259.4 | allocated_MB: 405.3 | reserved_MB: 12168.0 | inactive_split_MB: 126.3 +rank2 [2026-10-27 17:00:50,805] INFO : Step 15000/100000 rank2 | Train Loss: 12.4223 | Train Loss Unweighted: 4.6301 | Train Loss Components: Classification: 0.0132 | Classification_binary: 3.7975 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3484 | Regression_eta: 0.0765 | Regression_pt: 0.2883 | Regression_sin_phi: 0.0502 | Group Train Loss Unweighted (window mean): physical_pu: 3.8205 | physical_nopu: 4.8025 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2713 | Classification_binary: 3.7165 | Regression_cos_phi: 2.1658 | Regression_energy: 1.6845 | Regression_eta: 0.8671 | Regression_pt: 2.0629 | Regression_sin_phi: 1.4821 | LR: 3.80e-04 | DataLoad Time: 0.0011s | Model Forward Time: 3.2234s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0799 | max_loss: 0.0851 | max_backward: 0.1383 | max_optimizer_step: 0.1621 | max_tensorboard_logging: 2.1079 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5320.9 | uss_MB: 4324.4 | child_rss_MB: 10137.7 | allocated_MB: 394.1 | reserved_MB: 12168.0 | inactive_split_MB: 89.6 +rank2 [2026-10-27 17:04:12,219] INFO : Step 16000/100000 rank2 | Train Loss: 13.1559 | Train Loss Unweighted: 4.6950 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.6917 | Regression_cos_phi: 0.0473 | Regression_energy: 0.4330 | Regression_eta: 0.0771 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.8147 | physical_nopu: 4.7795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3624 | Classification_binary: 3.5051 | Regression_cos_phi: 1.7405 | Regression_energy: 2.6650 | Regression_eta: 0.8472 | Regression_pt: 3.5654 | Regression_sin_phi: 1.6650 | LR: 3.78e-04 | DataLoad Time: 0.0033s | Model Forward Time: 3.2746s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.0598 | max_loss: 0.0036 | max_backward: 0.1881 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 2.1012 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5304.0 | uss_MB: 4326.3 | child_rss_MB: 10027.9 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 88.9 +rank2 [2026-10-27 17:07:14,836] INFO : Step 17000/100000 rank2 | Train Loss: 13.6189 | Train Loss Unweighted: 4.8180 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7348 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4756 | Regression_eta: 0.0731 | Regression_pt: 0.4225 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.8095 | physical_nopu: 4.7681 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2888 | Classification_binary: 3.8011 | Regression_cos_phi: 2.0669 | Regression_energy: 2.8181 | Regression_eta: 0.7826 | Regression_pt: 3.6856 | Regression_sin_phi: 1.6795 | LR: 3.75e-04 | DataLoad Time: 0.0015s | Model Forward Time: 4.0410s | Phase Times: max_pre_step_barrier: 0.6520 | max_batch_to_device: 0.0829 | max_forward: 0.1388 | max_loss: 0.0036 | max_backward: 0.0636 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 2.2240 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5273.0 | uss_MB: 4329.2 | child_rss_MB: 9776.7 | allocated_MB: 394.3 | reserved_MB: 12168.0 | inactive_split_MB: 88.4 +rank2 [2026-10-27 17:10:19,258] INFO : Step 18000/100000 rank2 | Train Loss: 12.6032 | Train Loss Unweighted: 4.6266 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7426 | Regression_cos_phi: 0.0507 | Regression_energy: 0.3730 | Regression_eta: 0.0736 | Regression_pt: 0.3195 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7954 | physical_nopu: 4.7518 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2932 | Classification_binary: 3.8187 | Regression_cos_phi: 1.7542 | Regression_energy: 1.8051 | Regression_eta: 0.9110 | Regression_pt: 2.1616 | Regression_sin_phi: 1.9504 | LR: 3.72e-04 | DataLoad Time: 0.0011s | Model Forward Time: 4.2078s | Phase Times: max_pre_step_barrier: 0.0217 | max_batch_to_device: 0.0009 | max_forward: 0.1212 | max_loss: 0.0040 | max_backward: 0.1670 | max_optimizer_step: 0.1610 | max_tensorboard_logging: 3.0399 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5245.9 | uss_MB: 4331.6 | child_rss_MB: 9623.0 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 87.1 +rank2 [2026-10-27 17:13:21,531] INFO : Step 19000/100000 rank2 | Train Loss: 14.6123 | Train Loss Unweighted: 5.0036 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.6735 | Regression_cos_phi: 0.0398 | Regression_energy: 0.6019 | Regression_eta: 0.0612 | Regression_pt: 0.5669 | Regression_sin_phi: 0.0441 | Group Train Loss Unweighted (window mean): physical_pu: 3.7887 | physical_nopu: 4.7426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3602 | Classification_binary: 3.7257 | Regression_cos_phi: 1.6082 | Regression_energy: 2.2546 | Regression_eta: 0.6681 | Regression_pt: 2.9784 | Regression_sin_phi: 1.6167 | LR: 3.69e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.4964s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1593 | max_loss: 0.0094 | max_backward: 0.0749 | max_optimizer_step: 0.1658 | max_tensorboard_logging: 2.3024 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5218.9 | uss_MB: 4333.7 | child_rss_MB: 9466.4 | allocated_MB: 394.5 | reserved_MB: 12168.0 | inactive_split_MB: 103.1 +rank2 [2026-10-27 17:16:34,024] INFO : Step 20000/100000 rank2 | Train Loss: 13.1046 | Train Loss Unweighted: 4.7571 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.7799 | Regression_cos_phi: 0.0517 | Regression_energy: 0.4251 | Regression_eta: 0.0654 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0511 | Group Train Loss Unweighted (window mean): physical_pu: 3.7835 | physical_nopu: 4.7206 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2669 | Classification_binary: 3.8633 | Regression_cos_phi: 1.7281 | Regression_energy: 1.4832 | Regression_eta: 0.6035 | Regression_pt: 1.8608 | Regression_sin_phi: 1.5953 | LR: 3.66e-04 | DataLoad Time: 0.0031s | Model Forward Time: 3.2224s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0008 | max_forward: 0.1344 | max_loss: 0.0037 | max_backward: 0.0763 | max_optimizer_step: 0.1649 | max_tensorboard_logging: 1.9693 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5216.3 | uss_MB: 4336.2 | child_rss_MB: 9462.6 | allocated_MB: 392.8 | reserved_MB: 12168.0 | inactive_split_MB: 89.4 +rank2 [2026-10-27 17:16:34,968] INFO : Running validation on rank2 +rank2 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 17:18:27,486] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 17:18:27,879] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 17:18:53,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 17:18:54,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 17:18:54,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 17:18:55,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 17:18:55,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 17:18:56,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 17:18:56,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 17:18:57,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 17:19:07,108] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 17:19:07,312] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 17:19:07,733] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 17:19:22,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 17:19:23,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 17:19:23,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 17:19:24,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 17:19:24,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 17:19:24,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 17:19:25,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 17:19:25,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 17:19:35,914] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 17:19:36,428] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 17:19:36,813] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 17:19:52,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 17:19:52,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 17:19:52,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 17:19:53,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 17:19:53,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 17:19:54,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 17:19:54,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 17:19:55,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 17:20:05,105] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 17:23:25,419] INFO : Step 21000/100000 rank2 | Train Loss: 12.4320 | Train Loss Unweighted: 4.5235 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.6041 | Regression_cos_phi: 0.0444 | Regression_energy: 0.3902 | Regression_eta: 0.0812 | Regression_pt: 0.3383 | Regression_sin_phi: 0.0478 | Group Train Loss Unweighted (window mean): physical_pu: 3.7732 | physical_nopu: 4.7052 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4105 | Classification_binary: 3.5009 | Regression_cos_phi: 1.5771 | Regression_energy: 1.8774 | Regression_eta: 0.7924 | Regression_pt: 2.4542 | Regression_sin_phi: 1.7809 | LR: 3.62e-04 | DataLoad Time: 0.0027s | Model Forward Time: 3.7287s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.2105 | max_loss: 0.0056 | max_backward: 0.1347 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 2.3606 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5326.2 | uss_MB: 4454.3 | child_rss_MB: 9450.4 | allocated_MB: 394.6 | reserved_MB: 11062.0 | inactive_split_MB: 89.8 +rank2 [2026-10-27 17:26:14,701] INFO : Step 22000/100000 rank2 | Train Loss: 14.5232 | Train Loss Unweighted: 4.8893 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.6034 | Regression_cos_phi: 0.0476 | Regression_energy: 0.5714 | Regression_eta: 0.0738 | Regression_pt: 0.5351 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.7709 | physical_nopu: 4.6938 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3452 | Classification_binary: 3.6359 | Regression_cos_phi: 1.9333 | Regression_energy: 4.2320 | Regression_eta: 1.0688 | Regression_pt: 6.1211 | Regression_sin_phi: 1.7977 | LR: 3.59e-04 | DataLoad Time: 0.0031s | Model Forward Time: 2.6767s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0010 | max_forward: 0.0774 | max_loss: 0.0142 | max_backward: 0.0921 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.4442 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1027 | Memory: rss_MB: 5327.2 | uss_MB: 4458.5 | child_rss_MB: 9447.9 | allocated_MB: 394.2 | reserved_MB: 11488.0 | inactive_split_MB: 86.8 +rank2 [2026-10-27 17:29:50,014] INFO : Step 23000/100000 rank2 | Train Loss: 12.2140 | Train Loss Unweighted: 4.5907 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.7888 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3388 | Regression_eta: 0.0762 | Regression_pt: 0.2687 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.7690 | physical_nopu: 4.6810 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2201 | Classification_binary: 3.6028 | Regression_cos_phi: 1.7381 | Regression_energy: 1.8797 | Regression_eta: 0.8551 | Regression_pt: 1.9575 | Regression_sin_phi: 1.9316 | LR: 3.55e-04 | DataLoad Time: 0.0091s | Model Forward Time: 4.1197s | Phase Times: max_pre_step_barrier: 0.7590 | max_batch_to_device: 0.0007 | max_forward: 0.0763 | max_loss: 0.0189 | max_backward: 0.1117 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 2.0913 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5325.2 | uss_MB: 4459.8 | child_rss_MB: 9460.6 | allocated_MB: 394.0 | reserved_MB: 12220.0 | inactive_split_MB: 178.1 +rank2 [2026-10-27 17:33:22,189] INFO : Step 24000/100000 rank2 | Train Loss: 15.4786 | Train Loss Unweighted: 4.9908 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.5301 | Regression_cos_phi: 0.0467 | Regression_energy: 0.6682 | Regression_eta: 0.0624 | Regression_pt: 0.6223 | Regression_sin_phi: 0.0450 | Group Train Loss Unweighted (window mean): physical_pu: 3.7564 | physical_nopu: 4.6761 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3292 | Classification_binary: 3.8358 | Regression_cos_phi: 1.5641 | Regression_energy: 2.1275 | Regression_eta: 0.8181 | Regression_pt: 2.8027 | Regression_sin_phi: 1.7234 | LR: 3.51e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.9319s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0719 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1672 | max_tensorboard_logging: 1.8279 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5318.8 | uss_MB: 4462.5 | child_rss_MB: 9451.5 | allocated_MB: 393.8 | reserved_MB: 12220.0 | inactive_split_MB: 105.1 +rank2 [2026-10-27 17:36:26,065] INFO : Step 25000/100000 rank2 | Train Loss: 12.1246 | Train Loss Unweighted: 4.4783 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.7007 | Regression_cos_phi: 0.0551 | Regression_energy: 0.3196 | Regression_eta: 0.0694 | Regression_pt: 0.2642 | Regression_sin_phi: 0.0578 | Group Train Loss Unweighted (window mean): physical_pu: 3.7531 | physical_nopu: 4.6582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2320 | Classification_binary: 3.5463 | Regression_cos_phi: 1.7412 | Regression_energy: 1.2606 | Regression_eta: 0.6141 | Regression_pt: 1.5367 | Regression_sin_phi: 1.5386 | LR: 3.47e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.0302s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.1917 | max_loss: 0.0039 | max_backward: 0.1029 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 5304.8 | uss_MB: 4464.1 | child_rss_MB: 9445.8 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 86.4 +rank2 [2026-10-27 17:39:30,100] INFO : Step 26000/100000 rank2 | Train Loss: 12.6920 | Train Loss Unweighted: 4.5479 | Train Loss Components: Classification: 0.0188 | Classification_binary: 3.6003 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4034 | Regression_eta: 0.0734 | Regression_pt: 0.3564 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.7511 | physical_nopu: 4.6505 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3430 | Classification_binary: 3.5890 | Regression_cos_phi: 1.8327 | Regression_energy: 2.4845 | Regression_eta: 0.7481 | Regression_pt: 3.2226 | Regression_sin_phi: 1.3733 | LR: 3.43e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.0941s | Phase Times: max_pre_step_barrier: 0.0479 | max_batch_to_device: 0.0008 | max_forward: 0.1278 | max_loss: 0.0037 | max_backward: 0.0877 | max_optimizer_step: 0.1868 | max_tensorboard_logging: 1.8908 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5292.2 | uss_MB: 4467.2 | child_rss_MB: 9407.7 | allocated_MB: 395.0 | reserved_MB: 12220.0 | inactive_split_MB: 115.0 +rank2 [2026-10-27 17:42:32,922] INFO : Step 27000/100000 rank2 | Train Loss: 16.2758 | Train Loss Unweighted: 5.1421 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5608 | Regression_cos_phi: 0.0448 | Regression_energy: 0.7180 | Regression_eta: 0.0780 | Regression_pt: 0.6791 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.7443 | physical_nopu: 4.6426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3808 | Classification_binary: 3.6389 | Regression_cos_phi: 1.6172 | Regression_energy: 2.8796 | Regression_eta: 0.7141 | Regression_pt: 3.9881 | Regression_sin_phi: 1.7159 | LR: 3.39e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.0080s | Phase Times: max_pre_step_barrier: 0.0385 | max_batch_to_device: 0.0009 | max_forward: 0.0708 | max_loss: 0.0038 | max_backward: 0.0914 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.8878 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5283.2 | uss_MB: 4469.8 | child_rss_MB: 9400.6 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 87.9 +rank2 [2026-10-27 17:45:38,005] INFO : Step 28000/100000 rank2 | Train Loss: 11.9115 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6175 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3339 | Regression_eta: 0.0761 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.7393 | physical_nopu: 4.6288 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2269 | Classification_binary: 3.3108 | Regression_cos_phi: 1.7281 | Regression_energy: 1.6021 | Regression_eta: 0.6861 | Regression_pt: 1.9849 | Regression_sin_phi: 1.5080 | LR: 3.35e-04 | DataLoad Time: 0.0018s | Model Forward Time: 4.8024s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0008 | max_forward: 0.0886 | max_loss: 0.0038 | max_backward: 0.1193 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 3.6285 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 5274.2 | uss_MB: 4146.2 | child_rss_MB: 9355.1 | allocated_MB: 394.0 | reserved_MB: 12414.0 | inactive_split_MB: 106.5 +rank2 [2026-10-27 17:48:54,715] INFO : Step 29000/100000 rank2 | Train Loss: 13.9221 | Train Loss Unweighted: 4.6615 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.5075 | Regression_cos_phi: 0.0491 | Regression_energy: 0.5008 | Regression_eta: 0.0786 | Regression_pt: 0.4589 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7405 | physical_nopu: 4.6233 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4249 | Classification_binary: 3.3302 | Regression_cos_phi: 1.5136 | Regression_energy: 2.3437 | Regression_eta: 0.7770 | Regression_pt: 3.3343 | Regression_sin_phi: 1.5007 | LR: 3.30e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.3815s | Phase Times: max_pre_step_barrier: 0.1677 | max_batch_to_device: 0.0009 | max_forward: 0.0754 | max_loss: 0.0038 | max_backward: 0.0825 | max_optimizer_step: 0.1673 | max_tensorboard_logging: 2.1272 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5252.2 | uss_MB: 4453.2 | child_rss_MB: 9260.7 | allocated_MB: 394.9 | reserved_MB: 12414.0 | inactive_split_MB: 107.1 +rank2 [2026-10-27 17:51:57,834] INFO : Step 30000/100000 rank2 | Train Loss: 12.4988 | Train Loss Unweighted: 4.5639 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.6431 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3981 | Regression_eta: 0.0722 | Regression_pt: 0.3405 | Regression_sin_phi: 0.0498 | Group Train Loss Unweighted (window mean): physical_pu: 3.7342 | physical_nopu: 4.6125 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2583 | Classification_binary: 3.5742 | Regression_cos_phi: 1.9243 | Regression_energy: 1.6398 | Regression_eta: 0.6310 | Regression_pt: 2.0092 | Regression_sin_phi: 1.4984 | LR: 3.26e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.4592s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0762 | max_loss: 0.0136 | max_backward: 0.0854 | max_optimizer_step: 0.1722 | max_tensorboard_logging: 1.3349 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5242.2 | uss_MB: 4475.0 | child_rss_MB: 9193.8 | allocated_MB: 393.7 | reserved_MB: 12480.0 | inactive_split_MB: 89.7 +rank2 [2026-10-27 17:51:58,780] INFO : Running validation on rank2 +rank2 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 17:51:58,781] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 17:51:58,781] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 17:53:53,313] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 17:53:53,721] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 17:54:18,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 17:54:18,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 17:54:19,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 17:54:19,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 17:54:20,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 17:54:20,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 17:54:21,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 17:54:21,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 17:54:31,706] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 17:54:31,840] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 17:54:32,213] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 17:54:47,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 17:54:47,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 17:54:48,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 17:54:48,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 17:54:49,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 17:54:49,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 17:54:50,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 17:54:50,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 17:55:00,566] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 17:55:00,638] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 17:55:01,041] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 17:55:16,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 17:55:16,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 17:55:17,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 17:55:17,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 17:55:18,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 17:55:18,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 17:55:19,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 17:55:19,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 17:55:29,648] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-27 17:58:47,285] INFO : Step 31000/100000 rank2 | Train Loss: 12.0027 | Train Loss Unweighted: 4.4503 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.5779 | Regression_cos_phi: 0.0460 | Regression_energy: 0.3731 | Regression_eta: 0.0704 | Regression_pt: 0.3230 | Regression_sin_phi: 0.0451 | Group Train Loss Unweighted (window mean): physical_pu: 3.7262 | physical_nopu: 4.6083 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3500 | Classification_binary: 3.3429 | Regression_cos_phi: 1.8117 | Regression_energy: 1.8242 | Regression_eta: 0.5557 | Regression_pt: 2.4160 | Regression_sin_phi: 1.4007 | LR: 3.21e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.5858s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.1458 | max_loss: 0.0037 | max_backward: 0.1256 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 2.3030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7018.1 | uss_MB: 6249.8 | child_rss_MB: 9197.9 | allocated_MB: 394.8 | reserved_MB: 11168.0 | inactive_split_MB: 96.2 +rank2 [2026-10-27 18:01:54,932] INFO : Step 32000/100000 rank2 | Train Loss: 13.5217 | Train Loss Unweighted: 4.6918 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.5357 | Regression_cos_phi: 0.0443 | Regression_energy: 0.5169 | Regression_eta: 0.0728 | Regression_pt: 0.4649 | Regression_sin_phi: 0.0409 | Group Train Loss Unweighted (window mean): physical_pu: 3.7264 | physical_nopu: 4.6006 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3638 | Classification_binary: 3.6302 | Regression_cos_phi: 1.5291 | Regression_energy: 2.3989 | Regression_eta: 0.7203 | Regression_pt: 3.4137 | Regression_sin_phi: 0.7844 | LR: 3.16e-04 | DataLoad Time: 0.0079s | Model Forward Time: 2.3905s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.0795 | max_loss: 0.0917 | max_backward: 0.1851 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.1890 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5310.0 | uss_MB: 4562.1 | child_rss_MB: 9157.4 | allocated_MB: 394.9 | reserved_MB: 11240.0 | inactive_split_MB: 92.3 +rank2 [2026-10-27 18:04:57,678] INFO : Step 33000/100000 rank2 | Train Loss: 11.0126 | Train Loss Unweighted: 3.7465 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9928 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2905 | Regression_eta: 0.1058 | Regression_pt: 0.2472 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.7276 | physical_nopu: 4.5909 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3582 | Classification_binary: 2.9217 | Regression_cos_phi: 1.7205 | Regression_energy: 1.4993 | Regression_eta: 1.1512 | Regression_pt: 1.8901 | Regression_sin_phi: 1.5677 | LR: 3.12e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.8907s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0027 | max_forward: 0.1127 | max_loss: 0.0150 | max_backward: 0.3396 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.4660 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5295.8 | uss_MB: 4291.2 | child_rss_MB: 9057.9 | allocated_MB: 404.8 | reserved_MB: 11922.0 | inactive_split_MB: 135.4 +rank2 [2026-10-27 18:07:50,121] INFO : Step 34000/100000 rank2 | Train Loss: 14.2400 | Train Loss Unweighted: 4.7634 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5051 | Regression_cos_phi: 0.0450 | Regression_energy: 0.5627 | Regression_eta: 0.0631 | Regression_pt: 0.5251 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7180 | physical_nopu: 4.5858 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4019 | Classification_binary: 3.4069 | Regression_cos_phi: 1.4244 | Regression_energy: 2.7196 | Regression_eta: 0.5728 | Regression_pt: 3.8014 | Regression_sin_phi: 1.3043 | LR: 3.07e-04 | DataLoad Time: 0.0012s | Model Forward Time: 2.7416s | Phase Times: max_pre_step_barrier: 0.1802 | max_batch_to_device: 0.0765 | max_forward: 0.1650 | max_loss: 0.0036 | max_backward: 0.0935 | max_optimizer_step: 0.2788 | max_tensorboard_logging: 1.2621 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5285.5 | uss_MB: 4563.5 | child_rss_MB: 9059.3 | allocated_MB: 395.5 | reserved_MB: 11922.0 | inactive_split_MB: 99.3 +rank2 [2026-10-27 18:10:53,637] INFO : Step 35000/100000 rank2 | Train Loss: 11.4005 | Train Loss Unweighted: 4.4142 | Train Loss Components: Classification: 0.0091 | Classification_binary: 3.6972 | Regression_cos_phi: 0.0530 | Regression_energy: 0.2984 | Regression_eta: 0.0671 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.7147 | physical_nopu: 4.5802 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2042 | Classification_binary: 3.6006 | Regression_cos_phi: 2.1640 | Regression_energy: 1.3591 | Regression_eta: 0.7656 | Regression_pt: 1.6479 | Regression_sin_phi: 1.6051 | LR: 3.02e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.4163s | Phase Times: max_pre_step_barrier: 0.0682 | max_batch_to_device: 0.0008 | max_forward: 0.1243 | max_loss: 0.0038 | max_backward: 0.0883 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.1949 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5274.5 | uss_MB: 4566.4 | child_rss_MB: 9063.3 | allocated_MB: 393.8 | reserved_MB: 11922.0 | inactive_split_MB: 85.2 +rank2 [2026-10-27 18:13:50,577] INFO : Step 36000/100000 rank2 | Train Loss: 11.8438 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.5751 | Regression_cos_phi: 0.0431 | Regression_energy: 0.3555 | Regression_eta: 0.0641 | Regression_pt: 0.3210 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.7110 | physical_nopu: 4.5729 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3721 | Classification_binary: 3.5059 | Regression_cos_phi: 2.0967 | Regression_energy: 1.9993 | Regression_eta: 0.8654 | Regression_pt: 2.5807 | Regression_sin_phi: 2.0552 | LR: 2.97e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.4515s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0993 | max_loss: 0.0066 | max_backward: 0.1760 | max_optimizer_step: 0.1735 | max_tensorboard_logging: 1.2166 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 5259.5 | uss_MB: 4568.3 | child_rss_MB: 9032.9 | allocated_MB: 395.0 | reserved_MB: 11922.0 | inactive_split_MB: 95.2 +rank2 [2026-10-27 18:16:45,596] INFO : Step 37000/100000 rank2 | Train Loss: 15.1390 | Train Loss Unweighted: 4.8166 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4396 | Regression_cos_phi: 0.0489 | Regression_energy: 0.6142 | Regression_eta: 0.0886 | Regression_pt: 0.5621 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7102 | physical_nopu: 4.5673 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3505 | Classification_binary: 3.4287 | Regression_cos_phi: 1.7337 | Regression_energy: 2.3168 | Regression_eta: 0.6854 | Regression_pt: 2.9247 | Regression_sin_phi: 1.6659 | LR: 2.91e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.2788s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1889 | max_loss: 0.0036 | max_backward: 0.0899 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 1.0901 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 5243.2 | uss_MB: 4569.9 | child_rss_MB: 9022.4 | allocated_MB: 394.3 | reserved_MB: 11922.0 | inactive_split_MB: 107.1 +rank2 [2026-10-27 18:19:42,075] INFO : Step 38000/100000 rank2 | Train Loss: 11.6256 | Train Loss Unweighted: 4.4424 | Train Loss Components: Classification: 0.0117 | Classification_binary: 3.6501 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3369 | Regression_eta: 0.0664 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7131 | physical_nopu: 4.5682 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3357 | Classification_binary: 3.7048 | Regression_cos_phi: 1.7988 | Regression_energy: 1.6259 | Regression_eta: 0.6880 | Regression_pt: 1.8820 | Regression_sin_phi: 1.7730 | LR: 2.86e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.2500s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.0965 | max_loss: 0.0038 | max_backward: 0.1051 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5237.2 | uss_MB: 4571.0 | child_rss_MB: 9030.3 | allocated_MB: 393.3 | reserved_MB: 12082.0 | inactive_split_MB: 110.5 +rank2 [2026-10-27 18:22:37,812] INFO : Step 39000/100000 rank2 | Train Loss: 13.0208 | Train Loss Unweighted: 4.4928 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4758 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4360 | Regression_eta: 0.0793 | Regression_pt: 0.3879 | Regression_sin_phi: 0.0500 | Group Train Loss Unweighted (window mean): physical_pu: 3.7060 | physical_nopu: 4.5607 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3692 | Classification_binary: 3.4759 | Regression_cos_phi: 1.3737 | Regression_energy: 2.3323 | Regression_eta: 0.6575 | Regression_pt: 3.1739 | Regression_sin_phi: 1.9085 | LR: 2.81e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.2313s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.0864 | max_loss: 0.0038 | max_backward: 0.0877 | max_optimizer_step: 0.1700 | max_tensorboard_logging: 1.1418 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5224.7 | uss_MB: 4572.8 | child_rss_MB: 9008.2 | allocated_MB: 394.4 | reserved_MB: 12706.0 | inactive_split_MB: 84.9 +rank2 [2026-10-27 18:25:31,277] INFO : Step 40000/100000 rank2 | Train Loss: 14.7562 | Train Loss Unweighted: 4.9007 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5708 | Regression_cos_phi: 0.0459 | Regression_energy: 0.5954 | Regression_eta: 0.0747 | Regression_pt: 0.5553 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7006 | physical_nopu: 4.5579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2268 | Classification_binary: 3.5663 | Regression_cos_phi: 1.5398 | Regression_energy: 1.6209 | Regression_eta: 0.7244 | Regression_pt: 2.0347 | Regression_sin_phi: 1.4066 | LR: 2.76e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.3076s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.0814 | max_loss: 0.0614 | max_backward: 0.1417 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 1.1378 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5222.7 | uss_MB: 4573.6 | child_rss_MB: 8992.8 | allocated_MB: 393.7 | reserved_MB: 12706.0 | inactive_split_MB: 107.7 +rank2 [2026-10-27 18:25:32,320] INFO : Running validation on rank2 +rank2 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 18:27:23,858] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 18:27:24,220] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 18:27:48,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 18:27:48,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 18:27:49,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 18:27:49,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 18:27:50,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 18:27:50,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 18:27:51,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 18:27:51,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 18:28:01,713] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 18:28:01,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 18:28:02,236] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 18:28:17,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 18:28:17,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 18:28:18,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 18:28:18,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 18:28:19,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 18:28:19,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 18:28:20,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 18:28:20,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 18:28:30,777] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 18:28:30,989] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 18:28:31,367] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 18:28:46,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 18:28:47,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 18:28:47,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 18:28:48,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 18:28:48,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 18:28:49,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 18:28:49,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 18:28:50,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 18:29:00,105] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-27 18:32:27,442] INFO : Step 41000/100000 rank2 | Train Loss: 12.3609 | Train Loss Unweighted: 4.4817 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5838 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3837 | Regression_eta: 0.0731 | Regression_pt: 0.3287 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.7016 | physical_nopu: 4.5433 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3352 | Classification_binary: 3.7123 | Regression_cos_phi: 1.7298 | Regression_energy: 2.1244 | Regression_eta: 0.8973 | Regression_pt: 2.7051 | Regression_sin_phi: 1.6579 | LR: 2.70e-04 | DataLoad Time: 0.0009s | Model Forward Time: 4.5183s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0009 | max_forward: 0.1338 | max_loss: 0.0038 | max_backward: 0.1857 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 3.1120 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 7028.8 | uss_MB: 6392.1 | child_rss_MB: 8998.3 | allocated_MB: 394.0 | reserved_MB: 11364.0 | inactive_split_MB: 106.1 +rank2 [2026-10-27 18:35:19,857] INFO : Step 42000/100000 rank2 | Train Loss: 12.9790 | Train Loss Unweighted: 4.5354 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4379 | Regression_cos_phi: 0.0458 | Regression_energy: 0.4856 | Regression_eta: 0.0625 | Regression_pt: 0.4500 | Regression_sin_phi: 0.0370 | Group Train Loss Unweighted (window mean): physical_pu: 3.6959 | physical_nopu: 4.5474 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4074 | Classification_binary: 3.3100 | Regression_cos_phi: 1.1403 | Regression_energy: 2.0309 | Regression_eta: 0.6220 | Regression_pt: 2.6075 | Regression_sin_phi: 1.2448 | LR: 2.65e-04 | DataLoad Time: 0.0106s | Model Forward Time: 2.6333s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0009 | max_forward: 0.1375 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 1.3492 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 7016.8 | uss_MB: 5332.4 | child_rss_MB: 8994.4 | allocated_MB: 394.2 | reserved_MB: 12150.0 | inactive_split_MB: 99.3 +rank2 [2026-10-27 18:38:35,935] INFO : Step 43000/100000 rank2 | Train Loss: 10.9006 | Train Loss Unweighted: 3.7057 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.9740 | Regression_cos_phi: 0.0473 | Regression_energy: 0.2786 | Regression_eta: 0.1094 | Regression_pt: 0.2322 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6942 | physical_nopu: 4.5421 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3503 | Classification_binary: 2.9618 | Regression_cos_phi: 1.7329 | Regression_energy: 1.3845 | Regression_eta: 1.2125 | Regression_pt: 1.6897 | Regression_sin_phi: 1.6692 | LR: 2.59e-04 | DataLoad Time: 0.0020s | Model Forward Time: 5.2292s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0039 | max_forward: 0.1100 | max_loss: 0.0637 | max_backward: 0.2303 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 3.9250 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5285.0 | uss_MB: 4686.1 | child_rss_MB: 8875.5 | allocated_MB: 406.8 | reserved_MB: 12150.0 | inactive_split_MB: 152.7 +rank2 [2026-10-27 18:41:39,775] INFO : Step 44000/100000 rank2 | Train Loss: 14.8629 | Train Loss Unweighted: 4.8399 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4856 | Regression_cos_phi: 0.0438 | Regression_energy: 0.6070 | Regression_eta: 0.0739 | Regression_pt: 0.5644 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6933 | physical_nopu: 4.5331 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3239 | Classification_binary: 3.5814 | Regression_cos_phi: 1.2692 | Regression_energy: 2.0772 | Regression_eta: 0.6082 | Regression_pt: 2.8979 | Regression_sin_phi: 1.1989 | LR: 2.54e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.6839s | Phase Times: max_pre_step_barrier: 0.0851 | max_batch_to_device: 0.0008 | max_forward: 0.0946 | max_loss: 0.0041 | max_backward: 0.0964 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.3558 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5284.0 | uss_MB: 4679.1 | child_rss_MB: 8885.5 | allocated_MB: 394.8 | reserved_MB: 12150.0 | inactive_split_MB: 94.9 +rank2 [2026-10-27 18:44:46,984] INFO : Step 45000/100000 rank2 | Train Loss: 12.9535 | Train Loss Unweighted: 4.5658 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.6115 | Regression_cos_phi: 0.0537 | Regression_energy: 0.4117 | Regression_eta: 0.0846 | Regression_pt: 0.3413 | Regression_sin_phi: 0.0531 | Group Train Loss Unweighted (window mean): physical_pu: 3.6892 | physical_nopu: 4.5283 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1703 | Classification_binary: 3.6057 | Regression_cos_phi: 2.1425 | Regression_energy: 1.4320 | Regression_eta: 0.8280 | Regression_pt: 1.4266 | Regression_sin_phi: 1.7621 | LR: 2.48e-04 | DataLoad Time: 0.0028s | Model Forward Time: 2.4261s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1077 | max_loss: 0.0040 | max_backward: 0.1457 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 1.2404 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 5275.4 | uss_MB: 4680.1 | child_rss_MB: 8893.2 | allocated_MB: 394.0 | reserved_MB: 13366.0 | inactive_split_MB: 74.2 +rank2 [2026-10-27 18:48:06,241] INFO : Step 46000/100000 rank2 | Train Loss: 12.3783 | Train Loss Unweighted: 4.4476 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.5252 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3923 | Regression_eta: 0.0679 | Regression_pt: 0.3506 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6869 | physical_nopu: 4.5281 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4023 | Classification_binary: 3.4887 | Regression_cos_phi: 1.5272 | Regression_energy: 1.8844 | Regression_eta: 0.8933 | Regression_pt: 2.3364 | Regression_sin_phi: 1.4138 | LR: 2.43e-04 | DataLoad Time: 0.0028s | Model Forward Time: 2.4363s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1021 | max_loss: 0.0036 | max_backward: 0.1895 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 1.1894 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 5269.4 | uss_MB: 4681.7 | child_rss_MB: 8897.5 | allocated_MB: 395.1 | reserved_MB: 13366.0 | inactive_split_MB: 103.1 +rank2 [2026-10-27 18:51:16,142] INFO : Step 47000/100000 rank2 | Train Loss: 14.5142 | Train Loss Unweighted: 4.7631 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.3922 | Regression_cos_phi: 0.0407 | Regression_energy: 0.6220 | Regression_eta: 0.0662 | Regression_pt: 0.5862 | Regression_sin_phi: 0.0406 | Group Train Loss Unweighted (window mean): physical_pu: 3.6853 | physical_nopu: 4.5234 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3352 | Classification_binary: 3.5235 | Regression_cos_phi: 1.3484 | Regression_energy: 3.0400 | Regression_eta: 0.6103 | Regression_pt: 4.1774 | Regression_sin_phi: 1.1815 | LR: 2.37e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.5380s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0667 | max_forward: 0.1426 | max_loss: 0.0037 | max_backward: 0.0873 | max_optimizer_step: 0.2591 | max_tensorboard_logging: 1.1650 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5267.4 | uss_MB: 4682.3 | child_rss_MB: 8896.3 | allocated_MB: 394.9 | reserved_MB: 14166.0 | inactive_split_MB: 95.2 +rank2 [2026-10-27 18:54:12,871] INFO : Step 48000/100000 rank2 | Train Loss: 11.2030 | Train Loss Unweighted: 4.3343 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.6447 | Regression_cos_phi: 0.0521 | Regression_energy: 0.2822 | Regression_eta: 0.0657 | Regression_pt: 0.2293 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6844 | physical_nopu: 4.5168 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2054 | Classification_binary: 3.5356 | Regression_cos_phi: 1.7803 | Regression_energy: 1.1532 | Regression_eta: 0.5796 | Regression_pt: 1.4614 | Regression_sin_phi: 1.7715 | LR: 2.31e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.2610s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.0058 | max_backward: 0.0852 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1613 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5253.4 | uss_MB: 4683.3 | child_rss_MB: 8890.9 | allocated_MB: 393.9 | reserved_MB: 14166.0 | inactive_split_MB: 101.2 +rank2 [2026-10-27 18:57:19,782] INFO : Step 49000/100000 rank2 | Train Loss: 12.4372 | Train Loss Unweighted: 4.4483 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.5320 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3881 | Regression_eta: 0.0742 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6801 | physical_nopu: 4.5174 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3544 | Classification_binary: 3.4589 | Regression_cos_phi: 1.6502 | Regression_energy: 2.1924 | Regression_eta: 1.3777 | Regression_pt: 2.7320 | Regression_sin_phi: 1.6956 | LR: 2.26e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.4742s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0009 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1014 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 1.3340 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5240.2 | uss_MB: 4683.4 | child_rss_MB: 8815.5 | allocated_MB: 393.7 | reserved_MB: 14166.0 | inactive_split_MB: 100.8 +rank2 [2026-10-27 19:00:32,233] INFO : Step 50000/100000 rank2 | Train Loss: 13.3654 | Train Loss Unweighted: 4.5742 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4424 | Regression_cos_phi: 0.0426 | Regression_energy: 0.5013 | Regression_eta: 0.0683 | Regression_pt: 0.4573 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6814 | physical_nopu: 4.5132 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3769 | Classification_binary: 3.5013 | Regression_cos_phi: 1.1536 | Regression_energy: 2.1092 | Regression_eta: 0.5588 | Regression_pt: 3.0316 | Regression_sin_phi: 1.7352 | LR: 2.20e-04 | DataLoad Time: 0.0012s | Model Forward Time: 2.5868s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0010 | max_forward: 0.1892 | max_loss: 0.0115 | max_backward: 0.1333 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.2641 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0862 | Memory: rss_MB: 5234.2 | uss_MB: 4685.5 | child_rss_MB: 8779.3 | allocated_MB: 394.6 | reserved_MB: 14166.0 | inactive_split_MB: 91.2 +rank2 [2026-10-27 19:00:33,174] INFO : Running validation on rank2 +rank2 [2026-10-27 19:00:33,174] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 19:02:37,501] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 19:02:37,940] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 19:03:02,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 19:03:03,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 19:03:03,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 19:03:04,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 19:03:04,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 19:03:05,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 19:03:05,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 19:03:06,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 19:03:16,135] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 19:03:16,262] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 19:03:16,635] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 19:03:31,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 19:03:32,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 19:03:32,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 19:03:33,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 19:03:33,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 19:03:33,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 19:03:34,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 19:03:34,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 19:03:44,881] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 19:03:45,438] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 19:03:45,816] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 19:04:01,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 19:04:01,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 19:04:02,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 19:04:02,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 19:04:02,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 19:04:03,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 19:04:03,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 19:04:04,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 19:04:14,194] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-27 19:07:44,424] INFO : Step 51000/100000 rank2 | Train Loss: 12.0993 | Train Loss Unweighted: 4.3946 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5485 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3591 | Regression_eta: 0.0789 | Regression_pt: 0.2941 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6745 | physical_nopu: 4.5037 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2786 | Classification_binary: 3.3222 | Regression_cos_phi: 1.3282 | Regression_energy: 1.4729 | Regression_eta: 0.6229 | Regression_pt: 1.9876 | Regression_sin_phi: 1.4110 | LR: 2.14e-04 | DataLoad Time: 0.0097s | Model Forward Time: 3.3978s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0008 | max_forward: 0.1391 | max_loss: 0.0036 | max_backward: 0.1509 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 2.0683 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 7102.6 | uss_MB: 6555.6 | child_rss_MB: 8775.1 | allocated_MB: 393.6 | reserved_MB: 10952.0 | inactive_split_MB: 101.2 +rank2 [2026-10-27 19:10:55,345] INFO : Step 52000/100000 rank2 | Train Loss: 10.7195 | Train Loss Unweighted: 3.6688 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9553 | Regression_cos_phi: 0.0470 | Regression_energy: 0.2719 | Regression_eta: 0.1059 | Regression_pt: 0.2258 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6784 | physical_nopu: 4.5078 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3671 | Classification_binary: 2.9327 | Regression_cos_phi: 1.6932 | Regression_energy: 1.3667 | Regression_eta: 1.2442 | Regression_pt: 1.6771 | Regression_sin_phi: 1.6672 | LR: 2.09e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.5039s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0031 | max_forward: 0.1853 | max_loss: 0.0256 | max_backward: 0.2604 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 2.0239 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7092.6 | uss_MB: 6564.5 | child_rss_MB: 8773.5 | allocated_MB: 404.8 | reserved_MB: 11626.0 | inactive_split_MB: 256.4 +rank2 [2026-10-27 19:13:48,854] INFO : Step 53000/100000 rank2 | Train Loss: 11.7022 | Train Loss Unweighted: 4.3546 | Train Loss Components: Classification: 0.0123 | Classification_binary: 3.5594 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3321 | Regression_eta: 0.0730 | Regression_pt: 0.2795 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6748 | physical_nopu: 4.5005 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3086 | Classification_binary: 3.6005 | Regression_cos_phi: 2.0859 | Regression_energy: 1.8872 | Regression_eta: 0.9459 | Regression_pt: 2.3649 | Regression_sin_phi: 2.4402 | LR: 2.03e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.1501s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0954 | max_loss: 0.0037 | max_backward: 0.1065 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.9892 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 5255.7 | uss_MB: 4729.4 | child_rss_MB: 8773.6 | allocated_MB: 393.6 | reserved_MB: 11910.0 | inactive_split_MB: 100.6 +rank2 [2026-10-27 19:17:18,773] INFO : Step 54000/100000 rank2 | Train Loss: 12.5762 | Train Loss Unweighted: 4.3665 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0487 | Regression_energy: 0.4104 | Regression_eta: 0.0739 | Regression_pt: 0.3685 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6696 | physical_nopu: 4.5036 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3466 | Classification_binary: 3.4285 | Regression_cos_phi: 1.7713 | Regression_energy: 1.8404 | Regression_eta: 0.6593 | Regression_pt: 2.2670 | Regression_sin_phi: 1.5863 | LR: 1.97e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.3954s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.1022 | max_forward: 0.1953 | max_loss: 0.0037 | max_backward: 0.0975 | max_optimizer_step: 0.2772 | max_tensorboard_logging: 2.0527 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5248.7 | uss_MB: 4730.3 | child_rss_MB: 8744.8 | allocated_MB: 394.9 | reserved_MB: 11910.0 | inactive_split_MB: 118.2 +rank2 [2026-10-27 19:20:28,222] INFO : Step 55000/100000 rank2 | Train Loss: 13.4632 | Train Loss Unweighted: 4.6500 | Train Loss Components: Classification: 0.0134 | Classification_binary: 3.5428 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4921 | Regression_eta: 0.0679 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6720 | physical_nopu: 4.4919 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4004 | Classification_binary: 3.4272 | Regression_cos_phi: 1.6034 | Regression_energy: 2.3077 | Regression_eta: 0.6868 | Regression_pt: 3.0321 | Regression_sin_phi: 1.4826 | LR: 1.92e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.8026s | Phase Times: max_pre_step_barrier: 0.1980 | max_batch_to_device: 0.0007 | max_forward: 0.0889 | max_loss: 0.0037 | max_backward: 0.2091 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 2.3779 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5235.8 | uss_MB: 4729.6 | child_rss_MB: 8640.7 | allocated_MB: 393.8 | reserved_MB: 11910.0 | inactive_split_MB: 89.1 +rank2 [2026-10-27 19:23:30,605] INFO : Step 56000/100000 rank2 | Train Loss: 11.6078 | Train Loss Unweighted: 4.2962 | Train Loss Components: Classification: 0.0140 | Classification_binary: 3.4759 | Regression_cos_phi: 0.0438 | Regression_energy: 0.3484 | Regression_eta: 0.0671 | Regression_pt: 0.2977 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6692 | physical_nopu: 4.4908 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2892 | Classification_binary: 3.3879 | Regression_cos_phi: 1.4841 | Regression_energy: 1.4436 | Regression_eta: 0.5317 | Regression_pt: 1.8402 | Regression_sin_phi: 1.5683 | LR: 1.86e-04 | DataLoad Time: 0.0015s | Model Forward Time: 3.2300s | Phase Times: max_pre_step_barrier: 0.0941 | max_batch_to_device: 0.0008 | max_forward: 0.0930 | max_loss: 0.0038 | max_backward: 0.0988 | max_optimizer_step: 0.1756 | max_tensorboard_logging: 1.8920 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 5221.8 | uss_MB: 4732.9 | child_rss_MB: 8582.9 | allocated_MB: 393.5 | reserved_MB: 11910.0 | inactive_split_MB: 115.7 +rank2 [2026-10-27 19:26:52,902] INFO : Step 57000/100000 rank2 | Train Loss: 13.9252 | Train Loss Unweighted: 4.6468 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4640 | Regression_cos_phi: 0.0464 | Regression_energy: 0.5144 | Regression_eta: 0.0781 | Regression_pt: 0.4791 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6687 | physical_nopu: 4.4875 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3857 | Classification_binary: 3.2772 | Regression_cos_phi: 1.6116 | Regression_energy: 3.1064 | Regression_eta: 0.7592 | Regression_pt: 4.3295 | Regression_sin_phi: 1.4601 | LR: 1.81e-04 | DataLoad Time: 0.0027s | Model Forward Time: 4.6257s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.1223 | max_forward: 0.2094 | max_loss: 0.0420 | max_backward: 0.1628 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 2.9072 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 5216.8 | uss_MB: 4734.2 | child_rss_MB: 8511.2 | allocated_MB: 394.3 | reserved_MB: 12240.0 | inactive_split_MB: 101.4 +rank2 [2026-10-27 19:30:07,652] INFO : Step 58000/100000 rank2 | Train Loss: 11.5910 | Train Loss Unweighted: 4.3047 | Train Loss Components: Classification: 0.0106 | Classification_binary: 3.5490 | Regression_cos_phi: 0.0544 | Regression_energy: 0.3108 | Regression_eta: 0.0631 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0510 | Group Train Loss Unweighted (window mean): physical_pu: 3.6630 | physical_nopu: 4.4842 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2930 | Classification_binary: 3.4206 | Regression_cos_phi: 1.7709 | Regression_energy: 2.1148 | Regression_eta: 0.7795 | Regression_pt: 2.7801 | Regression_sin_phi: 1.7004 | LR: 1.75e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.5122s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.1080 | max_backward: 0.1964 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.2749 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 5209.3 | uss_MB: 4734.4 | child_rss_MB: 8432.6 | allocated_MB: 393.9 | reserved_MB: 12240.0 | inactive_split_MB: 121.9 +rank2 [2026-10-27 19:33:12,149] INFO : Step 59000/100000 rank2 | Train Loss: 12.2218 | Train Loss Unweighted: 4.4115 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5166 | Regression_cos_phi: 0.0458 | Regression_energy: 0.3794 | Regression_eta: 0.0771 | Regression_pt: 0.3280 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6650 | physical_nopu: 4.4788 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3364 | Classification_binary: 3.5262 | Regression_cos_phi: 1.8292 | Regression_energy: 1.7985 | Regression_eta: 0.6721 | Regression_pt: 2.2679 | Regression_sin_phi: 1.5954 | LR: 1.70e-04 | DataLoad Time: 0.1794s | Model Forward Time: 3.0433s | Phase Times: max_pre_step_barrier: 0.6889 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0499 | max_backward: 0.1549 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5201.3 | uss_MB: 4735.4 | child_rss_MB: 8387.3 | allocated_MB: 394.0 | reserved_MB: 12240.0 | inactive_split_MB: 135.3 +rank2 [2026-10-27 19:36:26,517] INFO : Step 60000/100000 rank2 | Train Loss: 14.3810 | Train Loss Unweighted: 4.7229 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.4146 | Regression_cos_phi: 0.0449 | Regression_energy: 0.5841 | Regression_eta: 0.0716 | Regression_pt: 0.5495 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.6604 | physical_nopu: 4.4865 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3896 | Classification_binary: 3.5972 | Regression_cos_phi: 1.4791 | Regression_energy: 3.7891 | Regression_eta: 0.9607 | Regression_pt: 5.3619 | Regression_sin_phi: 1.2447 | LR: 1.64e-04 | DataLoad Time: 0.0028s | Model Forward Time: 2.4563s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0849 | max_loss: 0.0036 | max_backward: 0.1478 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 1.2880 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5191.9 | uss_MB: 4495.0 | child_rss_MB: 8328.9 | allocated_MB: 394.8 | reserved_MB: 12240.0 | inactive_split_MB: 87.0 +rank2 [2026-10-27 19:36:27,558] INFO : Running validation on rank2 +rank2 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 19:38:18,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 19:38:19,028] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 19:38:43,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 19:38:44,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 19:38:44,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 19:38:45,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 19:38:45,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 19:38:46,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 19:38:46,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 19:38:46,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 19:38:56,978] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 19:38:57,241] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 19:38:57,611] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 19:39:12,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 19:39:13,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 19:39:13,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 19:39:14,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 19:39:14,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 19:39:15,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 19:39:15,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 19:39:16,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 19:39:26,042] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 19:39:26,170] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 19:39:26,533] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 19:39:41,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 19:39:42,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 19:39:42,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 19:39:42,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 19:39:43,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 19:39:43,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 19:39:44,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 19:39:44,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 19:39:54,750] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-27 19:43:33,544] INFO : Step 61000/100000 rank2 | Train Loss: 11.4262 | Train Loss Unweighted: 4.3706 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.6500 | Regression_cos_phi: 0.0531 | Regression_energy: 0.2956 | Regression_eta: 0.0675 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6553 | physical_nopu: 4.4780 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2236 | Classification_binary: 3.4753 | Regression_cos_phi: 2.0192 | Regression_energy: 1.6028 | Regression_eta: 0.5777 | Regression_pt: 1.9202 | Regression_sin_phi: 1.5328 | LR: 1.59e-04 | DataLoad Time: 0.0072s | Model Forward Time: 3.1992s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1116 | max_loss: 0.0036 | max_backward: 0.1506 | max_optimizer_step: 0.1828 | max_tensorboard_logging: 1.9738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 5784.0 | uss_MB: 5329.2 | child_rss_MB: 8302.4 | allocated_MB: 393.5 | reserved_MB: 11190.0 | inactive_split_MB: 103.1 +rank2 [2026-10-27 19:46:49,589] INFO : Step 62000/100000 rank2 | Train Loss: 10.7781 | Train Loss Unweighted: 3.6510 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9253 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2791 | Regression_eta: 0.1043 | Regression_pt: 0.2316 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3430 | Classification_binary: 2.9091 | Regression_cos_phi: 1.6739 | Regression_energy: 1.3709 | Regression_eta: 1.1629 | Regression_pt: 1.6759 | Regression_sin_phi: 1.6822 | LR: 1.54e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.6556s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0030 | max_forward: 0.1294 | max_loss: 0.0038 | max_backward: 0.2007 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.2115 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0496 | Memory: rss_MB: 5775.0 | uss_MB: 5052.2 | child_rss_MB: 8186.6 | allocated_MB: 404.4 | reserved_MB: 11190.0 | inactive_split_MB: 148.6 +rank2 [2026-10-27 19:50:02,443] INFO : Step 63000/100000 rank2 | Train Loss: 11.5766 | Train Loss Unweighted: 4.3655 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6000 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3204 | Regression_eta: 0.0727 | Regression_pt: 0.2610 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6589 | physical_nopu: 4.4771 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3044 | Classification_binary: 3.5975 | Regression_cos_phi: 1.8265 | Regression_energy: 1.5726 | Regression_eta: 0.7854 | Regression_pt: 1.7566 | Regression_sin_phi: 1.7940 | LR: 1.49e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.3896s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.1036 | max_loss: 0.0037 | max_backward: 0.1140 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2208 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 5773.0 | uss_MB: 5331.6 | child_rss_MB: 8138.8 | allocated_MB: 393.8 | reserved_MB: 11190.0 | inactive_split_MB: 93.0 +rank2 [2026-10-27 19:53:17,772] INFO : Step 64000/100000 rank2 | Train Loss: 13.2709 | Train Loss Unweighted: 4.5542 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.4295 | Regression_cos_phi: 0.0429 | Regression_energy: 0.4948 | Regression_eta: 0.0672 | Regression_pt: 0.4587 | Regression_sin_phi: 0.0436 | Group Train Loss Unweighted (window mean): physical_pu: 3.6551 | physical_nopu: 4.4714 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.5058 | Classification_binary: 3.4150 | Regression_cos_phi: 1.6683 | Regression_energy: 2.2459 | Regression_eta: 0.6493 | Regression_pt: 3.0730 | Regression_sin_phi: 1.4779 | LR: 1.43e-04 | DataLoad Time: 0.0009s | Model Forward Time: 2.3973s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0928 | max_loss: 0.0570 | max_backward: 0.1589 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 1.2035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 5289.6 | uss_MB: 4857.6 | child_rss_MB: 8120.4 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank2 [2026-10-27 19:56:23,053] INFO : Step 65000/100000 rank2 | Train Loss: 15.6652 | Train Loss Unweighted: 4.9833 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5112 | Regression_cos_phi: 0.0496 | Regression_energy: 0.6688 | Regression_eta: 0.0629 | Regression_pt: 0.6290 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6554 | physical_nopu: 4.4650 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3522 | Classification_binary: 3.5314 | Regression_cos_phi: 2.1393 | Regression_energy: 3.6279 | Regression_eta: 0.6267 | Regression_pt: 5.1204 | Regression_sin_phi: 1.8270 | LR: 1.38e-04 | DataLoad Time: 0.0015s | Model Forward Time: 2.7633s | Phase Times: max_pre_step_barrier: 0.1158 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0042 | max_backward: 0.1974 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.4176 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5282.6 | uss_MB: 4858.3 | child_rss_MB: 8025.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 124.9 +rank2 [2026-10-27 19:59:28,722] INFO : Step 66000/100000 rank2 | Train Loss: 12.0073 | Train Loss Unweighted: 4.3350 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4949 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3535 | Regression_eta: 0.0691 | Regression_pt: 0.3018 | Regression_sin_phi: 0.0501 | Group Train Loss Unweighted (window mean): physical_pu: 3.6526 | physical_nopu: 4.4657 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3171 | Classification_binary: 3.6958 | Regression_cos_phi: 2.0074 | Regression_energy: 1.5510 | Regression_eta: 0.7573 | Regression_pt: 1.9495 | Regression_sin_phi: 1.9009 | LR: 1.33e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.4469s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0891 | max_loss: 0.0538 | max_backward: 0.1468 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 1.2753 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5277.7 | uss_MB: 4858.9 | child_rss_MB: 7982.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 107.1 +rank2 [2026-10-27 20:02:51,910] INFO : Step 67000/100000 rank2 | Train Loss: 13.3177 | Train Loss Unweighted: 4.5554 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.4800 | Regression_cos_phi: 0.0500 | Regression_energy: 0.4690 | Regression_eta: 0.0673 | Regression_pt: 0.4253 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6538 | physical_nopu: 4.4665 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3789 | Classification_binary: 3.4344 | Regression_cos_phi: 1.8854 | Regression_energy: 2.7110 | Regression_eta: 0.9469 | Regression_pt: 3.3659 | Regression_sin_phi: 1.5684 | LR: 1.28e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.9380s | Phase Times: max_pre_step_barrier: 0.0792 | max_batch_to_device: 0.0009 | max_forward: 0.0886 | max_loss: 0.1057 | max_backward: 0.1987 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 2.6109 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 5271.7 | uss_MB: 4859.8 | child_rss_MB: 7952.8 | allocated_MB: 393.9 | reserved_MB: 11792.0 | inactive_split_MB: 111.5 +rank2 [2026-10-27 20:06:09,103] INFO : Step 68000/100000 rank2 | Train Loss: 11.1535 | Train Loss Unweighted: 4.2643 | Train Loss Components: Classification: 0.0109 | Classification_binary: 3.5138 | Regression_cos_phi: 0.0456 | Regression_energy: 0.3163 | Regression_eta: 0.0648 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6527 | physical_nopu: 4.4693 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2448 | Classification_binary: 3.5356 | Regression_cos_phi: 1.9404 | Regression_energy: 1.5746 | Regression_eta: 0.7390 | Regression_pt: 2.0371 | Regression_sin_phi: 1.6563 | LR: 1.24e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.3901s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1008 | max_loss: 0.0767 | max_backward: 0.1768 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.1759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5263.4 | uss_MB: 4860.3 | child_rss_MB: 7883.7 | allocated_MB: 394.5 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank2 [2026-10-27 20:09:12,492] INFO : Step 69000/100000 rank2 | Train Loss: 12.1728 | Train Loss Unweighted: 4.3418 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4719 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3660 | Regression_eta: 0.0763 | Regression_pt: 0.3115 | Regression_sin_phi: 0.0548 | Group Train Loss Unweighted (window mean): physical_pu: 3.6494 | physical_nopu: 4.4629 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3560 | Classification_binary: 3.4808 | Regression_cos_phi: 1.9652 | Regression_energy: 2.0185 | Regression_eta: 1.2127 | Regression_pt: 2.5000 | Regression_sin_phi: 2.1834 | LR: 1.19e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.2961s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.1094 | max_loss: 0.0147 | max_backward: 0.1167 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 1.1204 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5251.4 | uss_MB: 4861.5 | child_rss_MB: 7816.8 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 106.5 +rank2 [2026-10-27 20:12:13,498] INFO : Step 70000/100000 rank2 | Train Loss: 14.9156 | Train Loss Unweighted: 4.7730 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.3975 | Regression_cos_phi: 0.0487 | Regression_energy: 0.6183 | Regression_eta: 0.0638 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0448 | Group Train Loss Unweighted (window mean): physical_pu: 3.6488 | physical_nopu: 4.4584 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3152 | Classification_binary: 3.6660 | Regression_cos_phi: 1.7338 | Regression_energy: 2.8166 | Regression_eta: 0.6097 | Regression_pt: 3.7000 | Regression_sin_phi: 1.9252 | LR: 1.14e-04 | DataLoad Time: 0.0061s | Model Forward Time: 2.3875s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1038 | max_loss: 0.0858 | max_backward: 0.1871 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 1.1619 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5245.8 | uss_MB: 4860.8 | child_rss_MB: 7775.8 | allocated_MB: 395.0 | reserved_MB: 11792.0 | inactive_split_MB: 111.9 +rank2 [2026-10-27 20:12:14,504] INFO : Running validation on rank2 +rank2 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 20:14:09,433] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 20:14:09,865] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 20:14:34,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 20:14:35,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 20:14:35,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 20:14:36,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 20:14:36,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 20:14:36,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 20:14:37,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 20:14:37,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 20:14:47,934] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 20:14:47,942] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 20:14:48,316] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 20:15:03,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 20:15:03,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 20:15:04,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 20:15:04,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 20:15:05,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 20:15:05,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 20:15:06,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 20:15:06,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 20:15:16,534] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 20:15:17,039] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 20:15:17,439] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 20:15:32,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 20:15:33,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 20:15:33,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 20:15:34,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 20:15:34,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 20:15:34,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 20:15:35,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 20:15:35,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 20:15:45,803] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-27 20:19:21,751] INFO : Step 71000/100000 rank2 | Train Loss: 10.9922 | Train Loss Unweighted: 3.6772 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9201 | Regression_cos_phi: 0.0477 | Regression_energy: 0.2906 | Regression_eta: 0.1066 | Regression_pt: 0.2485 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6484 | physical_nopu: 4.4561 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3495 | Classification_binary: 2.9881 | Regression_cos_phi: 1.8318 | Regression_energy: 1.4121 | Regression_eta: 1.2016 | Regression_pt: 1.7381 | Regression_sin_phi: 1.7241 | LR: 1.10e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.5363s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0652 | max_forward: 0.2596 | max_loss: 0.0036 | max_backward: 0.3497 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 1.8772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 7279.8 | uss_MB: 6900.2 | child_rss_MB: 7804.4 | allocated_MB: 405.9 | reserved_MB: 11130.0 | inactive_split_MB: 135.5 +rank2 [2026-10-27 20:22:34,778] INFO : Step 72000/100000 rank2 | Train Loss: 14.4733 | Train Loss Unweighted: 4.6898 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3667 | Regression_cos_phi: 0.0458 | Regression_energy: 0.5948 | Regression_eta: 0.0650 | Regression_pt: 0.5567 | Regression_sin_phi: 0.0444 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4536 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3876 | Classification_binary: 3.2098 | Regression_cos_phi: 1.5627 | Regression_energy: 4.0997 | Regression_eta: 0.9288 | Regression_pt: 5.7381 | Regression_sin_phi: 1.6141 | LR: 1.05e-04 | DataLoad Time: 0.0010s | Model Forward Time: 2.5079s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1390 | max_loss: 0.0037 | max_backward: 0.1007 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.2476 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1281 | Memory: rss_MB: 5316.2 | uss_MB: 4936.1 | child_rss_MB: 7727.0 | allocated_MB: 393.8 | reserved_MB: 11706.0 | inactive_split_MB: 88.5 +rank2 [2026-10-27 20:25:34,428] INFO : Step 73000/100000 rank2 | Train Loss: 11.7903 | Train Loss Unweighted: 4.2561 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.5010 | Regression_cos_phi: 0.0578 | Regression_energy: 0.3092 | Regression_eta: 0.0644 | Regression_pt: 0.2565 | Regression_sin_phi: 0.0557 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4567 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3562 | Classification_binary: 3.7038 | Regression_cos_phi: 1.8320 | Regression_energy: 1.6105 | Regression_eta: 0.6348 | Regression_pt: 2.0162 | Regression_sin_phi: 2.5306 | LR: 1.01e-04 | DataLoad Time: 0.0027s | Model Forward Time: 2.6059s | Phase Times: max_pre_step_barrier: 0.1246 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0122 | max_backward: 0.1972 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.2348 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5307.2 | uss_MB: 4937.3 | child_rss_MB: 7690.0 | allocated_MB: 393.5 | reserved_MB: 11706.0 | inactive_split_MB: 93.2 +rank2 [2026-10-27 20:28:47,370] INFO : Step 74000/100000 rank2 | Train Loss: 11.6748 | Train Loss Unweighted: 4.2803 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4309 | Regression_cos_phi: 0.0428 | Regression_energy: 0.3572 | Regression_eta: 0.0695 | Regression_pt: 0.3177 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.6485 | physical_nopu: 4.4502 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3331 | Classification_binary: 3.4855 | Regression_cos_phi: 1.4391 | Regression_energy: 1.7639 | Regression_eta: 1.0151 | Regression_pt: 2.2944 | Regression_sin_phi: 1.6629 | LR: 9.68e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.0484s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1195 | max_loss: 0.0060 | max_backward: 0.2082 | max_optimizer_step: 0.1763 | max_tensorboard_logging: 1.8023 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5300.9 | uss_MB: 4938.3 | child_rss_MB: 7614.5 | allocated_MB: 394.3 | reserved_MB: 11706.0 | inactive_split_MB: 97.0 +rank2 [2026-10-27 20:31:56,443] INFO : Step 75000/100000 rank2 | Train Loss: 14.1604 | Train Loss Unweighted: 4.6445 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.4007 | Regression_cos_phi: 0.0489 | Regression_energy: 0.5577 | Regression_eta: 0.0640 | Regression_pt: 0.5112 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.6489 | physical_nopu: 4.4484 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3477 | Classification_binary: 3.3102 | Regression_cos_phi: 1.7223 | Regression_energy: 2.7314 | Regression_eta: 0.7122 | Regression_pt: 3.6622 | Regression_sin_phi: 2.2648 | LR: 9.27e-05 | DataLoad Time: 0.0029s | Model Forward Time: 2.9323s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0831 | max_loss: 0.0038 | max_backward: 0.2058 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.5751 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1279 | Memory: rss_MB: 5290.9 | uss_MB: 4938.0 | child_rss_MB: 7581.3 | allocated_MB: 394.4 | reserved_MB: 11706.0 | inactive_split_MB: 89.1 +rank2 [2026-10-27 20:34:57,733] INFO : Step 76000/100000 rank2 | Train Loss: 11.4149 | Train Loss Unweighted: 4.2984 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5224 | Regression_cos_phi: 0.0478 | Regression_energy: 0.3298 | Regression_eta: 0.0709 | Regression_pt: 0.2685 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6454 | physical_nopu: 4.4486 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3154 | Classification_binary: 3.7283 | Regression_cos_phi: 1.8503 | Regression_energy: 1.4932 | Regression_eta: 0.6260 | Regression_pt: 1.8755 | Regression_sin_phi: 1.3369 | LR: 8.88e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.1726s | Phase Times: max_pre_step_barrier: 0.0296 | max_batch_to_device: 0.0996 | max_forward: 0.1816 | max_loss: 0.0036 | max_backward: 0.0930 | max_optimizer_step: 0.2829 | max_tensorboard_logging: 1.7104 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1287 | Memory: rss_MB: 5286.0 | uss_MB: 4937.3 | child_rss_MB: 7586.8 | allocated_MB: 393.6 | reserved_MB: 12330.0 | inactive_split_MB: 97.7 +rank2 [2026-10-27 20:38:16,884] INFO : Step 77000/100000 rank2 | Train Loss: 12.8789 | Train Loss Unweighted: 4.4653 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4333 | Regression_cos_phi: 0.0478 | Regression_energy: 0.4517 | Regression_eta: 0.0624 | Regression_pt: 0.4079 | Regression_sin_phi: 0.0461 | Group Train Loss Unweighted (window mean): physical_pu: 3.6433 | physical_nopu: 4.4448 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3148 | Classification_binary: 3.4073 | Regression_cos_phi: 1.8967 | Regression_energy: 2.1218 | Regression_eta: 0.8918 | Regression_pt: 2.5761 | Regression_sin_phi: 1.6147 | LR: 8.50e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.0359s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.0831 | max_loss: 0.0514 | max_backward: 0.1524 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.8652 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5273.0 | uss_MB: 4939.8 | child_rss_MB: 7463.9 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 99.2 +rank2 [2026-10-27 20:41:21,661] INFO : Step 78000/100000 rank2 | Train Loss: 13.2420 | Train Loss Unweighted: 4.6345 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.5815 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4598 | Regression_eta: 0.0679 | Regression_pt: 0.4144 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6415 | physical_nopu: 4.4494 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3222 | Classification_binary: 3.4368 | Regression_cos_phi: 1.7933 | Regression_energy: 3.0380 | Regression_eta: 0.8082 | Regression_pt: 4.0648 | Regression_sin_phi: 1.2107 | LR: 8.13e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.0721s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0927 | max_loss: 0.0036 | max_backward: 0.2060 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.8450 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5261.0 | uss_MB: 4940.8 | child_rss_MB: 7383.4 | allocated_MB: 394.7 | reserved_MB: 12330.0 | inactive_split_MB: 95.1 +rank2 [2026-10-27 20:44:31,405] INFO : Step 79000/100000 rank2 | Train Loss: 11.5047 | Train Loss Unweighted: 4.2761 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4919 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3261 | Regression_eta: 0.0640 | Regression_pt: 0.2834 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6416 | physical_nopu: 4.4460 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3030 | Classification_binary: 3.8036 | Regression_cos_phi: 1.8635 | Regression_energy: 1.6477 | Regression_eta: 0.7048 | Regression_pt: 2.0327 | Regression_sin_phi: 1.4835 | LR: 7.78e-05 | DataLoad Time: 0.0031s | Model Forward Time: 3.2151s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0253 | max_forward: 0.1473 | max_loss: 0.0038 | max_backward: 0.0955 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 1.9569 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5248.8 | uss_MB: 4513.6 | child_rss_MB: 7321.5 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 120.0 +rank2 [2026-10-27 20:47:43,818] INFO : Step 80000/100000 rank2 | Train Loss: 15.3065 | Train Loss Unweighted: 4.8564 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4208 | Regression_cos_phi: 0.0455 | Regression_energy: 0.6415 | Regression_eta: 0.0773 | Regression_pt: 0.6092 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6398 | physical_nopu: 4.4406 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3232 | Classification_binary: 3.6089 | Regression_cos_phi: 1.6427 | Regression_energy: 2.7572 | Regression_eta: 0.9110 | Regression_pt: 3.7472 | Regression_sin_phi: 1.6512 | LR: 7.44e-05 | DataLoad Time: 0.0081s | Model Forward Time: 3.0643s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0010 | max_forward: 0.0840 | max_loss: 0.0039 | max_backward: 0.0916 | max_optimizer_step: 0.2630 | max_tensorboard_logging: 1.7339 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5238.8 | uss_MB: 4942.0 | child_rss_MB: 7231.1 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 141.7 +rank2 [2026-10-27 20:47:44,748] INFO : Running validation on rank2 +rank2 [2026-10-27 20:47:44,748] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 20:49:38,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 20:49:38,982] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 20:50:04,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 20:50:05,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 20:50:05,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 20:50:06,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 20:50:06,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 20:50:07,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 20:50:07,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 20:50:08,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 20:50:18,279] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 20:50:18,317] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 20:50:18,743] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 20:50:33,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 20:50:34,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 20:50:34,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 20:50:35,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 20:50:35,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 20:50:36,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 20:50:36,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 20:50:37,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 20:50:47,119] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 20:50:47,267] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 20:50:47,676] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 20:51:02,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 20:51:03,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 20:51:03,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 20:51:03,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 20:51:04,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 20:51:04,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 20:51:05,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 20:51:05,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 20:51:15,681] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-27 20:54:47,405] INFO : Step 81000/100000 rank2 | Train Loss: 10.7072 | Train Loss Unweighted: 3.6241 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.9080 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2710 | Regression_eta: 0.1078 | Regression_pt: 0.2269 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6413 | physical_nopu: 4.4395 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3866 | Classification_binary: 2.8562 | Regression_cos_phi: 1.6593 | Regression_energy: 1.3319 | Regression_eta: 1.2593 | Regression_pt: 1.6777 | Regression_sin_phi: 1.6015 | LR: 7.11e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.4725s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0033 | max_forward: 0.1984 | max_loss: 0.0038 | max_backward: 0.3244 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 1.8991 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 7314.5 | uss_MB: 7027.3 | child_rss_MB: 7215.1 | allocated_MB: 408.0 | reserved_MB: 11650.0 | inactive_split_MB: 261.6 +rank2 [2026-10-27 20:58:04,710] INFO : Step 82000/100000 rank2 | Train Loss: 12.9094 | Train Loss Unweighted: 4.6043 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5247 | Regression_cos_phi: 0.0417 | Regression_energy: 0.4765 | Regression_eta: 0.0603 | Regression_pt: 0.4449 | Regression_sin_phi: 0.0402 | Group Train Loss Unweighted (window mean): physical_pu: 3.6409 | physical_nopu: 4.4420 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4039 | Classification_binary: 3.2786 | Regression_cos_phi: 1.5439 | Regression_energy: 3.8598 | Regression_eta: 0.7675 | Regression_pt: 5.4473 | Regression_sin_phi: 1.2208 | LR: 6.80e-05 | DataLoad Time: 0.0011s | Model Forward Time: 3.1718s | Phase Times: max_pre_step_barrier: 0.0156 | max_batch_to_device: 0.0008 | max_forward: 0.1472 | max_loss: 0.0038 | max_backward: 0.1553 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 1.8498 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5342.3 | uss_MB: 5059.9 | child_rss_MB: 7159.0 | allocated_MB: 393.8 | reserved_MB: 11650.0 | inactive_split_MB: 92.9 +rank2 [2026-10-27 21:01:20,554] INFO : Step 83000/100000 rank2 | Train Loss: 11.2556 | Train Loss Unweighted: 4.2618 | Train Loss Components: Classification: 0.0094 | Classification_binary: 3.5167 | Regression_cos_phi: 0.0501 | Regression_energy: 0.3127 | Regression_eta: 0.0669 | Regression_pt: 0.2586 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6408 | physical_nopu: 4.4397 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2114 | Classification_binary: 3.2383 | Regression_cos_phi: 1.4288 | Regression_energy: 2.2743 | Regression_eta: 0.5242 | Regression_pt: 3.1929 | Regression_sin_phi: 1.5802 | LR: 6.51e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.5752s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0008 | max_forward: 0.1683 | max_loss: 0.0037 | max_backward: 0.1023 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.3519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 5336.3 | uss_MB: 5061.6 | child_rss_MB: 7095.7 | allocated_MB: 393.2 | reserved_MB: 11650.0 | inactive_split_MB: 95.2 +rank2 [2026-10-27 21:04:33,664] INFO : Step 84000/100000 rank2 | Train Loss: 11.5964 | Train Loss Unweighted: 4.2611 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4346 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3486 | Regression_eta: 0.0638 | Regression_pt: 0.3055 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6366 | physical_nopu: 4.4419 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3554 | Classification_binary: 3.4240 | Regression_cos_phi: 1.4317 | Regression_energy: 1.5500 | Regression_eta: 0.6014 | Regression_pt: 2.0793 | Regression_sin_phi: 1.3721 | LR: 6.23e-05 | DataLoad Time: 0.0008s | Model Forward Time: 4.9349s | Phase Times: max_pre_step_barrier: 0.2933 | max_batch_to_device: 0.0008 | max_forward: 0.1129 | max_loss: 0.0110 | max_backward: 0.1312 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 3.4426 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5330.3 | uss_MB: 5063.1 | child_rss_MB: 7029.9 | allocated_MB: 393.9 | reserved_MB: 11650.0 | inactive_split_MB: 95.6 +rank2 [2026-10-27 21:07:28,408] INFO : Step 85000/100000 rank2 | Train Loss: 14.1090 | Train Loss Unweighted: 4.6641 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.3775 | Regression_cos_phi: 0.0425 | Regression_energy: 0.5754 | Regression_eta: 0.0615 | Regression_pt: 0.5479 | Regression_sin_phi: 0.0420 | Group Train Loss Unweighted (window mean): physical_pu: 3.6325 | physical_nopu: 4.4336 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4169 | Classification_binary: 3.2938 | Regression_cos_phi: 1.4360 | Regression_energy: 3.2981 | Regression_eta: 0.7574 | Regression_pt: 4.5906 | Regression_sin_phi: 1.1042 | LR: 5.96e-05 | DataLoad Time: 0.0010s | Model Forward Time: 2.5402s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0496 | max_backward: 0.1488 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 1.3583 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 5323.1 | uss_MB: 5063.4 | child_rss_MB: 7037.3 | allocated_MB: 394.0 | reserved_MB: 11698.0 | inactive_split_MB: 93.1 +rank2 [2026-10-27 21:10:26,799] INFO : Step 86000/100000 rank2 | Train Loss: 11.7921 | Train Loss Unweighted: 4.3412 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.5823 | Regression_cos_phi: 0.0537 | Regression_energy: 0.3118 | Regression_eta: 0.0750 | Regression_pt: 0.2521 | Regression_sin_phi: 0.0550 | Group Train Loss Unweighted (window mean): physical_pu: 3.6404 | physical_nopu: 4.4337 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2730 | Classification_binary: 3.4559 | Regression_cos_phi: 1.7425 | Regression_energy: 1.3102 | Regression_eta: 0.6777 | Regression_pt: 1.6061 | Regression_sin_phi: 1.7424 | LR: 5.71e-05 | DataLoad Time: 0.0020s | Model Forward Time: 2.4341s | Phase Times: max_pre_step_barrier: 0.0233 | max_batch_to_device: 0.0898 | max_forward: 0.1824 | max_loss: 0.0038 | max_backward: 0.0938 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.9995 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1323 | Memory: rss_MB: 5319.1 | uss_MB: 4616.7 | child_rss_MB: 7006.7 | allocated_MB: 393.2 | reserved_MB: 11704.0 | inactive_split_MB: 96.0 +rank2 [2026-10-27 21:13:31,178] INFO : Step 87000/100000 rank2 | Train Loss: 12.0054 | Train Loss Unweighted: 4.3232 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.3964 | Regression_cos_phi: 0.0444 | Regression_energy: 0.4046 | Regression_eta: 0.0620 | Regression_pt: 0.3567 | Regression_sin_phi: 0.0418 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3712 | Classification_binary: 3.2704 | Regression_cos_phi: 1.6969 | Regression_energy: 2.0088 | Regression_eta: 0.6505 | Regression_pt: 2.5268 | Regression_sin_phi: 1.2566 | LR: 5.48e-05 | DataLoad Time: 0.0011s | Model Forward Time: 2.3332s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.1290 | max_loss: 0.0039 | max_backward: 0.0975 | max_optimizer_step: 0.1784 | max_tensorboard_logging: 1.1620 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5310.9 | uss_MB: 5064.7 | child_rss_MB: 7011.5 | allocated_MB: 394.7 | reserved_MB: 11704.0 | inactive_split_MB: 94.3 +rank2 [2026-10-27 21:16:33,434] INFO : Step 88000/100000 rank2 | Train Loss: 14.2663 | Train Loss Unweighted: 4.6673 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.4533 | Regression_cos_phi: 0.0523 | Regression_energy: 0.5342 | Regression_eta: 0.0666 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0508 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2923 | Classification_binary: 3.3915 | Regression_cos_phi: 2.2029 | Regression_energy: 1.9602 | Regression_eta: 0.6082 | Regression_pt: 2.7432 | Regression_sin_phi: 1.5094 | LR: 5.26e-05 | DataLoad Time: 0.0025s | Model Forward Time: 2.3215s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0008 | max_forward: 0.0997 | max_loss: 0.0039 | max_backward: 0.1038 | max_optimizer_step: 0.2517 | max_tensorboard_logging: 1.0175 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5309.9 | uss_MB: 5065.2 | child_rss_MB: 7012.4 | allocated_MB: 393.7 | reserved_MB: 11704.0 | inactive_split_MB: 103.2 +rank2 [2026-10-27 21:19:30,808] INFO : Step 89000/100000 rank2 | Train Loss: 11.9885 | Train Loss Unweighted: 4.3122 | Train Loss Components: Classification: 0.0138 | Classification_binary: 3.4837 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3438 | Regression_eta: 0.0687 | Regression_pt: 0.2984 | Regression_sin_phi: 0.0554 | Group Train Loss Unweighted (window mean): physical_pu: 3.6329 | physical_nopu: 4.4374 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3250 | Classification_binary: 3.5582 | Regression_cos_phi: 2.2415 | Regression_energy: 1.9161 | Regression_eta: 0.8524 | Regression_pt: 2.5532 | Regression_sin_phi: 2.0191 | LR: 5.06e-05 | DataLoad Time: 0.0007s | Model Forward Time: 2.2350s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0976 | max_loss: 0.0053 | max_backward: 0.1242 | max_optimizer_step: 0.1785 | max_tensorboard_logging: 1.0326 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 5307.5 | uss_MB: 5066.4 | child_rss_MB: 7006.7 | allocated_MB: 395.0 | reserved_MB: 13422.0 | inactive_split_MB: 109.9 +rank2 [2026-10-27 21:22:22,522] INFO : Step 90000/100000 rank2 | Train Loss: 10.5888 | Train Loss Unweighted: 3.6256 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9282 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2656 | Regression_eta: 0.1044 | Regression_pt: 0.2178 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6354 | physical_nopu: 4.4354 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3466 | Classification_binary: 2.9526 | Regression_cos_phi: 1.7318 | Regression_energy: 1.3239 | Regression_eta: 1.1871 | Regression_pt: 1.6128 | Regression_sin_phi: 1.7038 | LR: 4.88e-05 | DataLoad Time: 0.0021s | Model Forward Time: 4.1185s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0033 | max_forward: 0.1953 | max_loss: 0.0645 | max_backward: 0.8258 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.9975 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5307.5 | uss_MB: 5074.7 | child_rss_MB: 7021.2 | allocated_MB: 404.6 | reserved_MB: 13422.0 | inactive_split_MB: 208.4 +rank2 [2026-10-27 21:22:23,503] INFO : Running validation on rank2 +rank2 [2026-10-27 21:22:23,504] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 21:22:23,504] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 21:22:23,504] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 21:22:23,504] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 21:24:23,625] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 21:24:24,116] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 21:24:48,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 21:24:49,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 21:24:49,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 21:24:50,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 21:24:50,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 21:24:51,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 21:24:51,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 21:24:52,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 21:25:02,197] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 21:25:02,472] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 21:25:02,869] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 21:25:18,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 21:25:18,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 21:25:19,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 21:25:19,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 21:25:20,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 21:25:20,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 21:25:21,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 21:25:21,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 21:25:31,475] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 21:25:32,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 21:25:32,492] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 21:25:47,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 21:25:48,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 21:25:48,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 21:25:48,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 21:25:49,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 21:25:49,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 21:25:50,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 21:25:50,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 21:26:00,891] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 21:29:48,334] INFO : Step 91000/100000 rank2 | Train Loss: 11.9808 | Train Loss Unweighted: 4.3520 | Train Loss Components: Classification: 0.0128 | Classification_binary: 3.5260 | Regression_cos_phi: 0.0522 | Regression_energy: 0.3480 | Regression_eta: 0.0723 | Regression_pt: 0.2901 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4346 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3154 | Classification_binary: 3.6183 | Regression_cos_phi: 1.5533 | Regression_energy: 1.3938 | Regression_eta: 0.6192 | Regression_pt: 1.7753 | Regression_sin_phi: 1.5258 | LR: 4.71e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.6012s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1743 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.3267 | max_tensorboard_logging: 2.0733 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0716 | Memory: rss_MB: 7308.2 | uss_MB: 7066.1 | child_rss_MB: 7044.9 | allocated_MB: 393.7 | reserved_MB: 11392.0 | inactive_split_MB: 89.1 +rank2 [2026-10-27 21:33:08,656] INFO : Step 92000/100000 rank2 | Train Loss: 14.4974 | Train Loss Unweighted: 4.7629 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4460 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5917 | Regression_eta: 0.0739 | Regression_pt: 0.5467 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6297 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3415 | Classification_binary: 3.3999 | Regression_cos_phi: 1.6330 | Regression_energy: 3.5828 | Regression_eta: 0.6754 | Regression_pt: 4.8562 | Regression_sin_phi: 1.4280 | LR: 4.57e-05 | DataLoad Time: 0.0021s | Model Forward Time: 2.6194s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1620 | max_loss: 0.0037 | max_backward: 0.2478 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 1.1676 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0595 | Memory: rss_MB: 7302.2 | uss_MB: 6481.6 | child_rss_MB: 6997.3 | allocated_MB: 394.0 | reserved_MB: 11392.0 | inactive_split_MB: 90.6 +rank2 [2026-10-27 21:36:11,295] INFO : Step 93000/100000 rank2 | Train Loss: 13.1041 | Train Loss Unweighted: 4.5189 | Train Loss Components: Classification: 0.0119 | Classification_binary: 3.5265 | Regression_cos_phi: 0.0575 | Regression_energy: 0.4342 | Regression_eta: 0.0649 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6346 | physical_nopu: 4.4327 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2698 | Classification_binary: 3.3923 | Regression_cos_phi: 1.9080 | Regression_energy: 1.3613 | Regression_eta: 0.6707 | Regression_pt: 1.6013 | Regression_sin_phi: 1.5395 | LR: 4.43e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.4223s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1013 | max_loss: 0.1069 | max_backward: 0.2035 | max_optimizer_step: 0.1792 | max_tensorboard_logging: 1.1596 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 5379.1 | uss_MB: 5141.2 | child_rss_MB: 6951.6 | allocated_MB: 393.6 | reserved_MB: 13294.0 | inactive_split_MB: 89.2 +rank2 [2026-10-27 21:39:11,303] INFO : Step 94000/100000 rank2 | Train Loss: 11.5177 | Train Loss Unweighted: 4.2756 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4441 | Regression_cos_phi: 0.0413 | Regression_energy: 0.3503 | Regression_eta: 0.0686 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6308 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3695 | Classification_binary: 3.4009 | Regression_cos_phi: 1.5284 | Regression_energy: 1.2417 | Regression_eta: 0.6762 | Regression_pt: 1.6363 | Regression_sin_phi: 1.3271 | LR: 4.32e-05 | DataLoad Time: 0.0023s | Model Forward Time: 6.3162s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0191 | max_forward: 0.1721 | max_loss: 0.0188 | max_backward: 0.2028 | max_optimizer_step: 0.2895 | max_tensorboard_logging: 4.7123 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 5372.1 | uss_MB: 5157.2 | child_rss_MB: 6925.7 | allocated_MB: 394.1 | reserved_MB: 13294.0 | inactive_split_MB: 89.1 +rank2 [2026-10-27 21:42:20,694] INFO : Step 95000/100000 rank2 | Train Loss: 14.1207 | Train Loss Unweighted: 4.6477 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3920 | Regression_cos_phi: 0.0441 | Regression_energy: 0.5776 | Regression_eta: 0.0602 | Regression_pt: 0.5087 | Regression_sin_phi: 0.0499 | Group Train Loss Unweighted (window mean): physical_pu: 3.6267 | physical_nopu: 4.4321 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2881 | Classification_binary: 3.4749 | Regression_cos_phi: 1.3700 | Regression_energy: 1.1633 | Regression_eta: 0.6021 | Regression_pt: 1.4508 | Regression_sin_phi: 1.5450 | LR: 4.22e-05 | DataLoad Time: 0.0010s | Model Forward Time: 2.5044s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.0941 | max_loss: 0.0036 | max_backward: 0.1734 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 1.2811 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5361.0 | uss_MB: 5157.8 | child_rss_MB: 6928.3 | allocated_MB: 396.0 | reserved_MB: 14294.0 | inactive_split_MB: 91.1 +rank2 [2026-10-27 21:45:16,251] INFO : Step 96000/100000 rank2 | Train Loss: 11.5036 | Train Loss Unweighted: 4.2276 | Train Loss Components: Classification: 0.0097 | Classification_binary: 3.4897 | Regression_cos_phi: 0.0538 | Regression_energy: 0.2982 | Regression_eta: 0.0802 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6278 | physical_nopu: 4.4329 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2007 | Classification_binary: 3.5221 | Regression_cos_phi: 1.9408 | Regression_energy: 1.3037 | Regression_eta: 0.6282 | Regression_pt: 1.6312 | Regression_sin_phi: 1.9776 | LR: 4.14e-05 | DataLoad Time: 0.0023s | Model Forward Time: 2.3869s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0007 | max_forward: 0.0991 | max_loss: 0.0037 | max_backward: 0.1000 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5357.0 | uss_MB: 5158.4 | child_rss_MB: 6899.7 | allocated_MB: 393.0 | reserved_MB: 14294.0 | inactive_split_MB: 95.3 +rank2 [2026-10-27 21:48:34,573] INFO : Step 97000/100000 rank2 | Train Loss: 11.8793 | Train Loss Unweighted: 4.3044 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.4420 | Regression_cos_phi: 0.0454 | Regression_energy: 0.3663 | Regression_eta: 0.0711 | Regression_pt: 0.3156 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.6277 | physical_nopu: 4.4255 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3544 | Classification_binary: 3.3726 | Regression_cos_phi: 1.3869 | Regression_energy: 1.9024 | Regression_eta: 0.9248 | Regression_pt: 2.5338 | Regression_sin_phi: 1.5714 | LR: 4.08e-05 | DataLoad Time: 0.0018s | Model Forward Time: 2.5916s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1192 | max_loss: 0.0048 | max_backward: 0.1852 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 5356.0 | uss_MB: 5157.9 | child_rss_MB: 6903.2 | allocated_MB: 394.7 | reserved_MB: 14294.0 | inactive_split_MB: 98.1 +rank2 [2026-10-27 21:51:32,851] INFO : Step 98000/100000 rank2 | Train Loss: 12.9367 | Train Loss Unweighted: 4.4840 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0441 | Regression_energy: 0.4811 | Regression_eta: 0.0611 | Regression_pt: 0.4378 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3181 | Classification_binary: 3.6064 | Regression_cos_phi: 1.8787 | Regression_energy: 1.7900 | Regression_eta: 0.7704 | Regression_pt: 2.3156 | Regression_sin_phi: 1.6262 | LR: 4.04e-05 | DataLoad Time: 0.0017s | Model Forward Time: 2.8945s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1318 | max_loss: 0.0038 | max_backward: 0.1127 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.6868 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 5362.0 | uss_MB: 5160.3 | child_rss_MB: 6899.7 | allocated_MB: 394.5 | reserved_MB: 14294.0 | inactive_split_MB: 82.0 +rank2 [2026-10-27 21:54:35,739] INFO : Step 99000/100000 rank2 | Train Loss: 11.6408 | Train Loss Unweighted: 4.3204 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.5557 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3117 | Regression_eta: 0.0641 | Regression_pt: 0.2727 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.6292 | physical_nopu: 4.4284 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3743 | Classification_binary: 3.6769 | Regression_cos_phi: 1.9063 | Regression_energy: 1.5569 | Regression_eta: 0.6904 | Regression_pt: 2.0873 | Regression_sin_phi: 1.6903 | LR: 4.01e-05 | DataLoad Time: 0.0074s | Model Forward Time: 3.0815s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1731 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.8331 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 5358.0 | uss_MB: 5160.5 | child_rss_MB: 6886.0 | allocated_MB: 394.3 | reserved_MB: 14294.0 | inactive_split_MB: 113.7 +rank2 [2026-10-27 21:57:56,502] INFO : Step 100000/100000 rank2 | Train Loss: 10.8909 | Train Loss Unweighted: 3.6545 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9133 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2833 | Regression_eta: 0.1041 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6336 | physical_nopu: 4.4311 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3666 | Classification_binary: 2.9283 | Regression_cos_phi: 1.6668 | Regression_energy: 1.4543 | Regression_eta: 1.1631 | Regression_pt: 1.8588 | Regression_sin_phi: 1.6913 | LR: 4.00e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.7389s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0030 | max_forward: 0.2088 | max_loss: 0.0393 | max_backward: 0.3031 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.9276 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 5350.9 | uss_MB: 5163.3 | child_rss_MB: 6834.2 | allocated_MB: 405.7 | reserved_MB: 14294.0 | inactive_split_MB: 165.9 +rank2 [2026-10-27 21:57:57,460] INFO : Running validation on rank2 +rank2 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 21:59:51,634] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 21:59:52,049] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 22:00:18,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 22:00:19,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 22:00:19,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 22:00:20,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 22:00:20,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 22:00:21,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 22:00:21,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 22:00:21,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 22:00:31,948] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 22:00:32,061] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 22:00:32,465] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 22:00:48,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 22:00:48,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 22:00:49,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 22:00:49,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 22:00:50,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 22:00:50,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 22:00:50,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 22:00:51,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 22:01:01,515] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 22:01:01,522] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 22:01:01,928] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 22:01:16,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 22:01:17,397] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 22:01:17,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 22:01:18,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 22:01:18,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 22:01:19,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 22:01:19,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 22:01:20,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 22:01:30,035] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-27 22:01:30,980] INFO : Training completed. Total time on device 2: 350.244min diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.3 b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.3 new file mode 100644 index 0000000000000000000000000000000000000000..13d2430c62dd50f4ae1e4b7966a922b9fb798734 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.3 @@ -0,0 +1,851 @@ +rank3 [2026-10-27 16:10:10,451] INFO : Initializing model with seed=12345; process seed=12348 +rank3 [2026-10-27 16:10:10,452] INFO : configured dtype=torch.bfloat16 for autocast +rank3 [2026-10-27 16:10:37,169] INFO : Instantiating model +rank3 [2026-10-27 16:10:37,169] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank3 [2026-10-27 16:10:37,169] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank3 [2026-10-27 16:10:37,201] INFO : Input encoding initialization took 0.03s +rank3 [2026-10-27 16:10:37,201] INFO : nn0 parameters: 2130944 +rank3 [2026-10-27 16:10:37,201] INFO : Initializing shared backbone layers, num_convs=3 +rank3 [2026-10-27 16:10:37,201] INFO : Initializing backbone layer 0 +rank3 [2026-10-27 16:10:37,201] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-27 16:10:37,212] INFO : Initializing backbone layer 1 +rank3 [2026-10-27 16:10:37,212] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-27 16:10:37,223] INFO : Initializing backbone layer 2 +rank3 [2026-10-27 16:10:37,223] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-27 16:10:37,233] INFO : Backbone initialization took 0.03s +rank3 [2026-10-27 16:10:37,234] INFO : backbone parameters: 4733952 +rank3 [2026-10-27 16:10:37,234] INFO : Initializing output DNNs +rank3 [2026-10-27 16:10:37,268] INFO : Output DNNs initialization took 0.03s +rank3 [2026-10-27 16:10:37,268] INFO : backbone_mode=BackboneMode.SHARED +rank3 [2026-10-27 16:10:37,268] INFO : jagged_attention enabled=True +rank3 [2026-10-27 16:10:37,268] INFO : flash_attn_varlen enabled=True +rank3 [2026-10-27 16:10:37,268] INFO : task_queries enabled=False +rank3 [2026-10-27 16:10:37,268] INFO : classification_norm parameters: 1024 +rank3 [2026-10-27 16:10:37,268] INFO : regression_norm parameters: 1024 +rank3 [2026-10-27 16:10:37,268] INFO : classification_query parameters: 0 +rank3 [2026-10-27 16:10:37,268] INFO : regression_query parameters: 0 +rank3 [2026-10-27 16:10:37,268] INFO : classification_readout parameters: 0 +rank3 [2026-10-27 16:10:37,269] INFO : regression_readout parameters: 0 +rank3 [2026-10-27 16:10:37,269] INFO : output_mode=elementwise +rank3 [2026-10-27 16:10:37,269] INFO : set_decoder parameters: 0 +rank3 [2026-10-27 16:10:37,269] INFO : nn_binary_particle parameters: 264706 +rank3 [2026-10-27 16:10:37,269] INFO : nn_pid parameters: 268297 +rank3 [2026-10-27 16:10:37,269] INFO : nn_pt parameters: 2113544 +rank3 [2026-10-27 16:10:37,269] INFO : nn_eta parameters: 264706 +rank3 [2026-10-27 16:10:37,269] INFO : nn_sin_phi parameters: 264706 +rank3 [2026-10-27 16:10:37,269] INFO : nn_cos_phi parameters: 264706 +rank3 [2026-10-27 16:10:37,269] INFO : nn_energy parameters: 2113544 +rank3 [2026-10-27 16:10:37,270] INFO : Total MLPF parameters: 12421153 +rank3 [2026-10-27 16:10:37,270] INFO : MLPF __init__ done +rank3 [2026-10-27 16:10:37,270] INFO : Instantiated model +rank3 [2026-10-27 16:10:37,270] INFO : Moving model to device rank=3 +rank3 [2026-10-27 16:10:37,300] INFO : Moved model to device rank=3 +rank3 [2026-10-27 16:10:37,302] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank3 [2026-10-27 16:10:37,302] INFO : Configured model for SyncBatchNorm rank=3 +rank3 [2026-10-27 16:10:37,493] INFO : Configured model for DistributedDataParallel rank=3 +rank3 [2026-10-27 16:10:37,593] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +167 module.nn_energy.nn.7.0.bias ... 0 +168 module.nn_energy.nn.7.2.weight ... 0 +169 module.nn_energy.nn.7.2.bias ... 0 +170 module.nn_energy.nn.7.4.weight ... 0 +171 module.nn_energy.nn.7.4.bias ... 0 + +[172 rows x 3 columns] +rank3 [2026-10-27 16:10:37,593] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=512, out_features=512, bias=True) + ) + (norm0): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=512, out_features=512, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank3 [2026-10-27 16:10:37,594] INFO : Trainable parameters: 12421153 +rank3 [2026-10-27 16:10:37,594] INFO : Non-trainable parameters: 0 +rank3 [2026-10-27 16:10:37,594] INFO : Total parameters: 12421153 +rank3 [2026-10-27 16:10:37,597] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.mha.in_proj_weight 786432 0 + module.backbone.0.mha.in_proj_bias 1536 0 +module.backbone.0.mha.out_proj.weight 262144 0 + module.backbone.0.mha.out_proj.bias 512 0 + module.backbone.0.norm0.weight 512 0 + module.backbone.0.norm0.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm1.bias 512 0 + module.backbone.0.seq.0.weight 262144 0 + module.backbone.0.seq.0.bias 512 0 + module.backbone.0.seq.2.weight 262144 0 + module.backbone.0.seq.2.bias 512 0 + module.backbone.1.mha.in_proj_weight 786432 0 + module.backbone.1.mha.in_proj_bias 1536 0 +module.backbone.1.mha.out_proj.weight 262144 0 + module.backbone.1.mha.out_proj.bias 512 0 + module.backbone.1.norm0.weight 512 0 + module.backbone.1.norm0.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm1.bias 512 0 + module.backbone.1.seq.0.weight 262144 0 + module.backbone.1.seq.0.bias 512 0 + module.backbone.1.seq.2.weight 262144 0 + module.backbone.1.seq.2.bias 512 0 + module.backbone.2.mha.in_proj_weight 786432 0 + module.backbone.2.mha.in_proj_bias 1536 0 +module.backbone.2.mha.out_proj.weight 262144 0 + module.backbone.2.mha.out_proj.bias 512 0 + module.backbone.2.norm0.weight 512 0 + module.backbone.2.norm0.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm1.bias 512 0 + module.backbone.2.seq.0.weight 262144 0 + module.backbone.2.seq.0.bias 512 0 + module.backbone.2.seq.2.weight 262144 0 + module.backbone.2.seq.2.bias 512 0 + module.classification_norm.weight 512 0 + module.classification_norm.bias 512 0 + module.regression_norm.weight 512 0 + module.regression_norm.bias 512 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank3 [2026-10-27 16:10:37,597] INFO : Getting dataloaders +rank3 [2026-10-27 16:10:37,597] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:43,311] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:44,039] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:44,835] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank3 [2026-10-27 16:10:44,835] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:45,826] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:46,993] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:48,018] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank3 [2026-10-27 16:10:48,018] INFO : Creating WeightedMixIterator with 2 data loaders. +rank3 [2026-10-27 16:10:48,018] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:48,176] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:48,334] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:48,491] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank3 [2026-10-27 16:10:48,491] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:48,655] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:48,820] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:48,956] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank3 [2026-10-27 16:10:48,956] INFO : Creating InterleavedIterator with 2 data loaders. +rank3 [2026-10-27 16:11:16,693] INFO : Got dataloaders +rank3 [2026-10-27 16:11:16,693] INFO : loader split=train rank=3 len=232058 +rank3 [2026-10-27 16:11:16,693] INFO : loader split=valid rank=3 len=48 +rank3 [2026-10-27 16:11:16,693] INFO : Starting training from step 1 to 100000 on rank 3 of 8 +rank3 [2026-10-27 16:11:16,693] INFO : Creating train_iterator +rank3 [2026-10-27 16:11:16,693] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank3 [2026-10-27 16:11:30,809] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank3 [2026-10-27 16:11:45,800] INFO : Created train_iterator +rank3 [2026-10-27 16:12:23,645] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.123548031449318, 'Classification': 0.2298217851272784, 'Regression_pt': 0.7194055180624127, 'Regression_eta': 0.4619348940253258, 'Regression_sin_phi': 0.15005176125559955, 'Regression_cos_phi': 0.14165307599119842, 'Regression_energy': 1.0569261094555258}, weights={'Classification_binary': 1.0, 'Classification': 22.293569564819336, 'Regression_pt': 7.12191915512085, 'Regression_eta': 11.091493606567383, 'Regression_sin_phi': 34.14520263671875, 'Regression_cos_phi': 36.16969299316406, 'Regression_energy': 4.847593307495117} +rank3 [2026-10-27 16:14:48,546] INFO : Step 1000/100000 rank3 | Train Loss: 15.9738 | Train Loss Unweighted: 5.5021 | Train Loss Components: Classification: 0.0191 | Classification_binary: 4.1436 | Regression_cos_phi: 0.0516 | Regression_energy: 0.6035 | Regression_eta: 0.0819 | Regression_pt: 0.5500 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 5.2846 | physical_nopu: 6.1663 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4256 | Classification_binary: 4.1749 | Regression_cos_phi: 1.8547 | Regression_energy: 4.3512 | Regression_eta: 0.9120 | Regression_pt: 5.9439 | Regression_sin_phi: 1.7225 | LR: 4.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9579s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0013 | max_forward: 0.0194 | max_loss: 0.0039 | max_backward: 0.0308 | max_optimizer_step: 0.1391 | max_tensorboard_logging: 1.0816 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0030 | Memory: rss_MB: 4094.8 | uss_MB: 2885.6 | child_rss_MB: 10815.8 | allocated_MB: 395.5 | reserved_MB: 11198.0 | inactive_split_MB: 104.9 +rank3 [2026-10-27 16:17:31,218] INFO : Step 2000/100000 rank3 | Train Loss: 17.2752 | Train Loss Unweighted: 5.6831 | Train Loss Components: Classification: 0.0157 | Classification_binary: 4.0887 | Regression_cos_phi: 0.0506 | Regression_energy: 0.7236 | Regression_eta: 0.0696 | Regression_pt: 0.6798 | Regression_sin_phi: 0.0552 | Group Train Loss Unweighted (window mean): physical_pu: 4.4021 | physical_nopu: 5.4797 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3976 | Classification_binary: 4.0256 | Regression_cos_phi: 1.7002 | Regression_energy: 3.6470 | Regression_eta: 0.6797 | Regression_pt: 5.0366 | Regression_sin_phi: 2.2199 | LR: 4.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7791s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0250 | max_loss: 0.0036 | max_backward: 0.0657 | max_optimizer_step: 0.1397 | max_tensorboard_logging: 0.8949 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4097.7 | uss_MB: 2895.6 | child_rss_MB: 10834.7 | allocated_MB: 394.2 | reserved_MB: 11198.0 | inactive_split_MB: 90.1 +rank3 [2026-10-27 16:20:19,985] INFO : Step 3000/100000 rank3 | Train Loss: 14.0295 | Train Loss Unweighted: 5.1258 | Train Loss Components: Classification: 0.0148 | Classification_binary: 4.0405 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4749 | Regression_eta: 0.0746 | Regression_pt: 0.4205 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 4.1913 | physical_nopu: 5.3261 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3431 | Classification_binary: 4.0284 | Regression_cos_phi: 1.5490 | Regression_energy: 2.3601 | Regression_eta: 0.8908 | Regression_pt: 2.8832 | Regression_sin_phi: 1.4604 | LR: 3.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.4929s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1067 | max_loss: 0.0036 | max_backward: 0.0365 | max_optimizer_step: 0.1415 | max_tensorboard_logging: 1.1098 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0093 | Memory: rss_MB: 4102.6 | uss_MB: 2907.4 | child_rss_MB: 10837.4 | allocated_MB: 395.4 | reserved_MB: 11784.0 | inactive_split_MB: 88.5 +rank3 [2026-10-27 16:23:08,050] INFO : Step 4000/100000 rank3 | Train Loss: 11.7244 | Train Loss Unweighted: 3.9925 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.1718 | Regression_cos_phi: 0.0487 | Regression_energy: 0.3212 | Regression_eta: 0.1125 | Regression_pt: 0.2729 | Regression_sin_phi: 0.0492 | Group Train Loss Unweighted (window mean): physical_pu: 4.0838 | physical_nopu: 5.2370 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3557 | Classification_binary: 3.1672 | Regression_cos_phi: 1.8623 | Regression_energy: 1.5225 | Regression_eta: 1.2135 | Regression_pt: 1.8801 | Regression_sin_phi: 1.6739 | LR: 3.99e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.2687s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0037 | max_forward: 0.0726 | max_loss: 0.0066 | max_backward: 0.1670 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4108.6 | uss_MB: 2925.8 | child_rss_MB: 10838.2 | allocated_MB: 406.4 | reserved_MB: 11904.0 | inactive_split_MB: 179.6 +rank3 [2026-10-27 16:25:55,983] INFO : Step 5000/100000 rank3 | Train Loss: 13.1313 | Train Loss Unweighted: 4.9481 | Train Loss Components: Classification: 0.0138 | Classification_binary: 4.0144 | Regression_cos_phi: 0.0518 | Regression_energy: 0.3902 | Regression_eta: 0.0928 | Regression_pt: 0.3380 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 4.0057 | physical_nopu: 5.1582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3317 | Classification_binary: 4.0251 | Regression_cos_phi: 1.8770 | Regression_energy: 2.0025 | Regression_eta: 1.0023 | Regression_pt: 2.5301 | Regression_sin_phi: 1.5615 | LR: 3.98e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.3573s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0196 | max_loss: 0.0037 | max_backward: 0.0324 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 1.3896 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0068 | Memory: rss_MB: 4091.1 | uss_MB: 2927.1 | child_rss_MB: 10673.2 | allocated_MB: 393.9 | reserved_MB: 11904.0 | inactive_split_MB: 90.0 +rank3 [2026-10-27 16:28:58,362] INFO : Step 6000/100000 rank3 | Train Loss: 14.5226 | Train Loss Unweighted: 5.0639 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.8601 | Regression_cos_phi: 0.0525 | Regression_energy: 0.5359 | Regression_eta: 0.0694 | Regression_pt: 0.4825 | Regression_sin_phi: 0.0459 | Group Train Loss Unweighted (window mean): physical_pu: 3.9666 | physical_nopu: 5.0829 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3640 | Classification_binary: 3.9123 | Regression_cos_phi: 2.0889 | Regression_energy: 2.3387 | Regression_eta: 0.7774 | Regression_pt: 3.0959 | Regression_sin_phi: 1.6481 | LR: 3.97e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.5989s | Phase Times: max_pre_step_barrier: 0.0280 | max_batch_to_device: 0.0008 | max_forward: 0.0311 | max_loss: 0.0037 | max_backward: 0.0629 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 2.6604 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0119 | Memory: rss_MB: 4078.1 | uss_MB: 2934.4 | child_rss_MB: 10495.3 | allocated_MB: 394.6 | reserved_MB: 12524.0 | inactive_split_MB: 109.0 +rank3 [2026-10-27 16:31:51,869] INFO : Step 7000/100000 rank3 | Train Loss: 13.9772 | Train Loss Unweighted: 5.0398 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.9636 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4671 | Regression_eta: 0.0797 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0537 | Group Train Loss Unweighted (window mean): physical_pu: 3.9390 | physical_nopu: 5.0316 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2288 | Classification_binary: 3.9795 | Regression_cos_phi: 1.9269 | Regression_energy: 1.5038 | Regression_eta: 0.6911 | Regression_pt: 1.9161 | Regression_sin_phi: 1.7104 | LR: 3.96e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.1733s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0243 | max_loss: 0.0036 | max_backward: 0.0295 | max_optimizer_step: 0.2429 | max_tensorboard_logging: 2.1406 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 4070.6 | uss_MB: 2937.8 | child_rss_MB: 10370.1 | allocated_MB: 393.9 | reserved_MB: 12524.0 | inactive_split_MB: 107.5 +rank3 [2026-10-27 16:35:03,086] INFO : Step 8000/100000 rank3 | Train Loss: 13.6161 | Train Loss Unweighted: 4.9650 | Train Loss Components: Classification: 0.0187 | Classification_binary: 3.9275 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4519 | Regression_eta: 0.0615 | Regression_pt: 0.4056 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.9031 | physical_nopu: 4.9878 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3825 | Classification_binary: 4.0520 | Regression_cos_phi: 1.7831 | Regression_energy: 2.2166 | Regression_eta: 0.6745 | Regression_pt: 2.8283 | Regression_sin_phi: 1.6637 | LR: 3.94e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.9574s | Phase Times: max_pre_step_barrier: 0.0692 | max_batch_to_device: 0.0893 | max_forward: 0.1169 | max_loss: 0.0037 | max_backward: 0.0488 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 1.9012 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4069.6 | uss_MB: 2945.4 | child_rss_MB: 10376.1 | allocated_MB: 393.8 | reserved_MB: 12524.0 | inactive_split_MB: 105.1 +rank3 [2026-10-27 16:38:07,476] INFO : Step 9000/100000 rank3 | Train Loss: 15.1606 | Train Loss Unweighted: 5.0855 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.7493 | Regression_cos_phi: 0.0474 | Regression_energy: 0.5909 | Regression_eta: 0.0749 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.8927 | physical_nopu: 4.9579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3784 | Classification_binary: 3.5798 | Regression_cos_phi: 1.6565 | Regression_energy: 3.0211 | Regression_eta: 0.8376 | Regression_pt: 3.9698 | Regression_sin_phi: 1.6269 | LR: 3.93e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.8504s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0007 | max_forward: 0.0386 | max_loss: 0.0041 | max_backward: 0.0582 | max_optimizer_step: 0.1496 | max_tensorboard_logging: 1.9225 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 4056.6 | uss_MB: 2948.9 | child_rss_MB: 10378.8 | allocated_MB: 395.8 | reserved_MB: 12524.0 | inactive_split_MB: 89.5 +rank3 [2026-10-27 16:41:18,312] INFO : Step 10000/100000 rank3 | Train Loss: 12.2525 | Train Loss Unweighted: 4.6725 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8667 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3374 | Regression_eta: 0.0750 | Regression_pt: 0.2772 | Regression_sin_phi: 0.0525 | Group Train Loss Unweighted (window mean): physical_pu: 3.8762 | physical_nopu: 4.9262 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2579 | Classification_binary: 3.8773 | Regression_cos_phi: 2.1027 | Regression_energy: 1.8614 | Regression_eta: 0.7669 | Regression_pt: 2.3040 | Regression_sin_phi: 2.0913 | LR: 3.91e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.1908s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0277 | max_loss: 0.0035 | max_backward: 0.0304 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 2.2107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4024.3 | uss_MB: 2953.8 | child_rss_MB: 10382.4 | allocated_MB: 393.6 | reserved_MB: 12524.0 | inactive_split_MB: 89.8 +rank3 [2026-10-27 16:41:19,251] INFO : Running validation on rank3 +rank3 [2026-10-27 16:41:19,251] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 16:43:24,651] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:43:25,081] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 16:43:52,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 16:43:52,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 16:43:53,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 16:43:53,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 16:43:54,521] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 16:43:55,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 16:43:55,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 16:43:56,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 16:44:06,035] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-27 16:44:06,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:44:06,938] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 16:44:22,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 16:44:22,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 16:44:23,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 16:44:23,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 16:44:23,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 16:44:24,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 16:44:24,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 16:44:25,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 16:44:35,401] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 16:44:35,782] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:44:36,273] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 16:44:51,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 16:44:51,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 16:44:52,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 16:44:52,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 16:44:53,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 16:44:53,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 16:44:54,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 16:44:54,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 16:45:04,490] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 16:48:51,455] INFO : Step 11000/100000 rank3 | Train Loss: 13.5331 | Train Loss Unweighted: 4.8490 | Train Loss Components: Classification: 0.0178 | Classification_binary: 3.8332 | Regression_cos_phi: 0.0514 | Regression_energy: 0.4326 | Regression_eta: 0.0715 | Regression_pt: 0.3906 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.8629 | physical_nopu: 4.8860 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4630 | Classification_binary: 3.8328 | Regression_cos_phi: 1.9295 | Regression_energy: 2.3194 | Regression_eta: 0.8174 | Regression_pt: 3.1622 | Regression_sin_phi: 1.9451 | LR: 3.89e-04 | DataLoad Time: 0.0027s | Model Forward Time: 3.1301s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0010 | max_forward: 0.0938 | max_loss: 0.0036 | max_backward: 0.1077 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.9405 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6748.2 | uss_MB: 5693.6 | child_rss_MB: 10378.2 | allocated_MB: 394.1 | reserved_MB: 11016.0 | inactive_split_MB: 102.9 +rank3 [2026-10-27 16:51:46,972] INFO : Step 12000/100000 rank3 | Train Loss: 16.0880 | Train Loss Unweighted: 5.2955 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.7102 | Regression_cos_phi: 0.0409 | Regression_energy: 0.7300 | Regression_eta: 0.0621 | Regression_pt: 0.6963 | Regression_sin_phi: 0.0392 | Group Train Loss Unweighted (window mean): physical_pu: 3.8522 | physical_nopu: 4.8676 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3512 | Classification_binary: 3.4758 | Regression_cos_phi: 0.9305 | Regression_energy: 5.2226 | Regression_eta: 0.5329 | Regression_pt: 7.5102 | Regression_sin_phi: 1.0342 | LR: 3.87e-04 | DataLoad Time: 0.0026s | Model Forward Time: 2.9528s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1050 | max_forward: 0.2073 | max_loss: 0.0038 | max_backward: 0.1040 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 1.6163 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6734.2 | uss_MB: 5695.7 | child_rss_MB: 10317.1 | allocated_MB: 394.0 | reserved_MB: 11384.0 | inactive_split_MB: 88.0 +rank3 [2026-10-27 16:54:47,130] INFO : Step 13000/100000 rank3 | Train Loss: 12.2526 | Train Loss Unweighted: 4.6994 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.8775 | Regression_cos_phi: 0.0480 | Regression_energy: 0.3426 | Regression_eta: 0.0688 | Regression_pt: 0.2970 | Regression_sin_phi: 0.0539 | Group Train Loss Unweighted (window mean): physical_pu: 3.8404 | physical_nopu: 4.8415 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1892 | Classification_binary: 3.7660 | Regression_cos_phi: 1.5392 | Regression_energy: 1.4821 | Regression_eta: 0.5933 | Regression_pt: 2.0364 | Regression_sin_phi: 1.3575 | LR: 3.85e-04 | DataLoad Time: 0.0074s | Model Forward Time: 3.3121s | Phase Times: max_pre_step_barrier: 0.0143 | max_batch_to_device: 0.0010 | max_forward: 0.0740 | max_loss: 0.0036 | max_backward: 0.0639 | max_optimizer_step: 0.2642 | max_tensorboard_logging: 2.1016 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 5345.9 | uss_MB: 4318.0 | child_rss_MB: 10308.5 | allocated_MB: 394.7 | reserved_MB: 12168.0 | inactive_split_MB: 111.5 +rank3 [2026-10-27 16:57:46,633] INFO : Step 14000/100000 rank3 | Train Loss: 11.1328 | Train Loss Unweighted: 3.8032 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.0595 | Regression_cos_phi: 0.0494 | Regression_energy: 0.2823 | Regression_eta: 0.1077 | Regression_pt: 0.2396 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.8281 | physical_nopu: 4.8178 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3577 | Classification_binary: 2.9599 | Regression_cos_phi: 1.7826 | Regression_energy: 1.3318 | Regression_eta: 1.1701 | Regression_pt: 1.6518 | Regression_sin_phi: 1.6317 | LR: 3.83e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.7658s | Phase Times: max_pre_step_barrier: 0.1064 | max_batch_to_device: 0.0033 | max_forward: 0.1130 | max_loss: 0.0036 | max_backward: 0.2788 | max_optimizer_step: 0.1663 | max_tensorboard_logging: 2.1758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1099 | Memory: rss_MB: 5335.9 | uss_MB: 4329.8 | child_rss_MB: 10259.4 | allocated_MB: 405.3 | reserved_MB: 12168.0 | inactive_split_MB: 126.3 +rank3 [2026-10-27 17:00:50,804] INFO : Step 15000/100000 rank3 | Train Loss: 12.4223 | Train Loss Unweighted: 4.6301 | Train Loss Components: Classification: 0.0132 | Classification_binary: 3.7975 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3484 | Regression_eta: 0.0765 | Regression_pt: 0.2883 | Regression_sin_phi: 0.0502 | Group Train Loss Unweighted (window mean): physical_pu: 3.8205 | physical_nopu: 4.8025 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3610 | Classification_binary: 3.7965 | Regression_cos_phi: 1.9806 | Regression_energy: 1.2861 | Regression_eta: 0.8381 | Regression_pt: 1.6621 | Regression_sin_phi: 1.4224 | LR: 3.80e-04 | DataLoad Time: 0.0015s | Model Forward Time: 3.2229s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0799 | max_loss: 0.0851 | max_backward: 0.1383 | max_optimizer_step: 0.1621 | max_tensorboard_logging: 2.1079 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5320.9 | uss_MB: 4324.4 | child_rss_MB: 10137.7 | allocated_MB: 394.1 | reserved_MB: 12168.0 | inactive_split_MB: 89.6 +rank3 [2026-10-27 17:04:12,219] INFO : Step 16000/100000 rank3 | Train Loss: 13.1559 | Train Loss Unweighted: 4.6950 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.6917 | Regression_cos_phi: 0.0473 | Regression_energy: 0.4330 | Regression_eta: 0.0771 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.8147 | physical_nopu: 4.7795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3066 | Classification_binary: 3.6995 | Regression_cos_phi: 1.5036 | Regression_energy: 2.0671 | Regression_eta: 0.9493 | Regression_pt: 2.7584 | Regression_sin_phi: 1.4435 | LR: 3.78e-04 | DataLoad Time: 0.0028s | Model Forward Time: 3.2749s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.0598 | max_loss: 0.0036 | max_backward: 0.1881 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 2.1012 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5304.0 | uss_MB: 4326.3 | child_rss_MB: 10027.9 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 88.9 +rank3 [2026-10-27 17:07:14,836] INFO : Step 17000/100000 rank3 | Train Loss: 13.6189 | Train Loss Unweighted: 4.8180 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7348 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4756 | Regression_eta: 0.0731 | Regression_pt: 0.4225 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.8095 | physical_nopu: 4.7681 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2677 | Classification_binary: 3.7476 | Regression_cos_phi: 1.7657 | Regression_energy: 2.5009 | Regression_eta: 0.7648 | Regression_pt: 3.1061 | Regression_sin_phi: 1.9125 | LR: 3.75e-04 | DataLoad Time: 0.0012s | Model Forward Time: 4.0411s | Phase Times: max_pre_step_barrier: 0.6520 | max_batch_to_device: 0.0829 | max_forward: 0.1388 | max_loss: 0.0036 | max_backward: 0.0636 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 2.2240 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5273.0 | uss_MB: 4329.2 | child_rss_MB: 9776.7 | allocated_MB: 394.3 | reserved_MB: 12168.0 | inactive_split_MB: 88.4 +rank3 [2026-10-27 17:10:19,258] INFO : Step 18000/100000 rank3 | Train Loss: 12.6032 | Train Loss Unweighted: 4.6266 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7426 | Regression_cos_phi: 0.0507 | Regression_energy: 0.3730 | Regression_eta: 0.0736 | Regression_pt: 0.3195 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7954 | physical_nopu: 4.7518 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3208 | Classification_binary: 3.9050 | Regression_cos_phi: 2.5709 | Regression_energy: 2.4368 | Regression_eta: 1.0803 | Regression_pt: 2.7918 | Regression_sin_phi: 2.1724 | LR: 3.72e-04 | DataLoad Time: 0.0020s | Model Forward Time: 4.2068s | Phase Times: max_pre_step_barrier: 0.0217 | max_batch_to_device: 0.0009 | max_forward: 0.1212 | max_loss: 0.0040 | max_backward: 0.1670 | max_optimizer_step: 0.1610 | max_tensorboard_logging: 3.0399 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5245.9 | uss_MB: 4331.6 | child_rss_MB: 9623.0 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 87.1 +rank3 [2026-10-27 17:13:21,531] INFO : Step 19000/100000 rank3 | Train Loss: 14.6123 | Train Loss Unweighted: 5.0036 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.6735 | Regression_cos_phi: 0.0398 | Regression_energy: 0.6019 | Regression_eta: 0.0612 | Regression_pt: 0.5669 | Regression_sin_phi: 0.0441 | Group Train Loss Unweighted (window mean): physical_pu: 3.7887 | physical_nopu: 4.7426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3146 | Classification_binary: 3.6637 | Regression_cos_phi: 1.4241 | Regression_energy: 2.0043 | Regression_eta: 0.6789 | Regression_pt: 2.5849 | Regression_sin_phi: 1.4282 | LR: 3.69e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.4963s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1593 | max_loss: 0.0094 | max_backward: 0.0749 | max_optimizer_step: 0.1658 | max_tensorboard_logging: 2.3024 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5218.9 | uss_MB: 4333.7 | child_rss_MB: 9466.4 | allocated_MB: 394.5 | reserved_MB: 12168.0 | inactive_split_MB: 103.1 +rank3 [2026-10-27 17:16:34,024] INFO : Step 20000/100000 rank3 | Train Loss: 13.1046 | Train Loss Unweighted: 4.7571 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.7799 | Regression_cos_phi: 0.0517 | Regression_energy: 0.4251 | Regression_eta: 0.0654 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0511 | Group Train Loss Unweighted (window mean): physical_pu: 3.7835 | physical_nopu: 4.7206 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3426 | Classification_binary: 3.7290 | Regression_cos_phi: 1.6107 | Regression_energy: 1.7008 | Regression_eta: 0.5341 | Regression_pt: 2.1090 | Regression_sin_phi: 1.9008 | LR: 3.66e-04 | DataLoad Time: 0.0026s | Model Forward Time: 3.2227s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0008 | max_forward: 0.1344 | max_loss: 0.0037 | max_backward: 0.0763 | max_optimizer_step: 0.1649 | max_tensorboard_logging: 1.9693 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5216.3 | uss_MB: 4336.2 | child_rss_MB: 9462.6 | allocated_MB: 392.8 | reserved_MB: 12168.0 | inactive_split_MB: 89.4 +rank3 [2026-10-27 17:16:34,962] INFO : Running validation on rank3 +rank3 [2026-10-27 17:16:34,962] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 17:16:34,962] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 17:16:34,962] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 17:16:34,962] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 17:18:27,486] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 17:18:27,879] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 17:18:53,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 17:18:54,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 17:18:54,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 17:18:55,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 17:18:55,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 17:18:56,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 17:18:56,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 17:18:56,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 17:19:06,992] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 17:19:07,312] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 17:19:07,733] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 17:19:22,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 17:19:23,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 17:19:23,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 17:19:24,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 17:19:24,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 17:19:25,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 17:19:25,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 17:19:25,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 17:19:35,994] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 17:19:36,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 17:19:36,812] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 17:19:52,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 17:19:52,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 17:19:53,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 17:19:53,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 17:19:54,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 17:19:54,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 17:19:55,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 17:19:55,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 17:20:05,654] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 17:23:25,419] INFO : Step 21000/100000 rank3 | Train Loss: 12.4320 | Train Loss Unweighted: 4.5235 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.6041 | Regression_cos_phi: 0.0444 | Regression_energy: 0.3902 | Regression_eta: 0.0812 | Regression_pt: 0.3383 | Regression_sin_phi: 0.0478 | Group Train Loss Unweighted (window mean): physical_pu: 3.7732 | physical_nopu: 4.7052 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3551 | Classification_binary: 3.7656 | Regression_cos_phi: 1.8193 | Regression_energy: 1.8333 | Regression_eta: 1.0941 | Regression_pt: 2.2790 | Regression_sin_phi: 1.7134 | LR: 3.62e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.7288s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.2105 | max_loss: 0.0056 | max_backward: 0.1347 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 2.3606 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5326.2 | uss_MB: 4454.3 | child_rss_MB: 9450.4 | allocated_MB: 394.6 | reserved_MB: 11062.0 | inactive_split_MB: 89.8 +rank3 [2026-10-27 17:26:14,701] INFO : Step 22000/100000 rank3 | Train Loss: 14.5232 | Train Loss Unweighted: 4.8893 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.6034 | Regression_cos_phi: 0.0476 | Regression_energy: 0.5714 | Regression_eta: 0.0738 | Regression_pt: 0.5351 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.7709 | physical_nopu: 4.6938 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3392 | Classification_binary: 3.5052 | Regression_cos_phi: 1.5683 | Regression_energy: 2.6532 | Regression_eta: 0.7365 | Regression_pt: 3.6790 | Regression_sin_phi: 1.2463 | LR: 3.59e-04 | DataLoad Time: 0.0027s | Model Forward Time: 2.6768s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0010 | max_forward: 0.0774 | max_loss: 0.0142 | max_backward: 0.0921 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.4442 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1027 | Memory: rss_MB: 5327.2 | uss_MB: 4458.5 | child_rss_MB: 9447.9 | allocated_MB: 394.2 | reserved_MB: 11488.0 | inactive_split_MB: 86.8 +rank3 [2026-10-27 17:29:50,014] INFO : Step 23000/100000 rank3 | Train Loss: 12.2140 | Train Loss Unweighted: 4.5907 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.7888 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3388 | Regression_eta: 0.0762 | Regression_pt: 0.2687 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.7690 | physical_nopu: 4.6810 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2546 | Classification_binary: 3.7671 | Regression_cos_phi: 1.8915 | Regression_energy: 1.5840 | Regression_eta: 0.8685 | Regression_pt: 1.9129 | Regression_sin_phi: 2.2810 | LR: 3.55e-04 | DataLoad Time: 0.0098s | Model Forward Time: 4.1189s | Phase Times: max_pre_step_barrier: 0.7590 | max_batch_to_device: 0.0007 | max_forward: 0.0763 | max_loss: 0.0189 | max_backward: 0.1117 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 2.0913 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5325.2 | uss_MB: 4459.8 | child_rss_MB: 9460.6 | allocated_MB: 394.0 | reserved_MB: 12220.0 | inactive_split_MB: 178.1 +rank3 [2026-10-27 17:33:22,189] INFO : Step 24000/100000 rank3 | Train Loss: 15.4786 | Train Loss Unweighted: 4.9908 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.5301 | Regression_cos_phi: 0.0467 | Regression_energy: 0.6682 | Regression_eta: 0.0624 | Regression_pt: 0.6223 | Regression_sin_phi: 0.0450 | Group Train Loss Unweighted (window mean): physical_pu: 3.7564 | physical_nopu: 4.6761 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4201 | Classification_binary: 3.4248 | Regression_cos_phi: 1.9946 | Regression_energy: 2.8145 | Regression_eta: 0.6385 | Regression_pt: 3.8695 | Regression_sin_phi: 1.8791 | LR: 3.51e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.9310s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0719 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1672 | max_tensorboard_logging: 1.8279 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5318.8 | uss_MB: 4462.5 | child_rss_MB: 9451.5 | allocated_MB: 393.8 | reserved_MB: 12220.0 | inactive_split_MB: 105.1 +rank3 [2026-10-27 17:36:26,064] INFO : Step 25000/100000 rank3 | Train Loss: 12.1246 | Train Loss Unweighted: 4.4783 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.7007 | Regression_cos_phi: 0.0551 | Regression_energy: 0.3196 | Regression_eta: 0.0694 | Regression_pt: 0.2642 | Regression_sin_phi: 0.0578 | Group Train Loss Unweighted (window mean): physical_pu: 3.7531 | physical_nopu: 4.6582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2783 | Classification_binary: 3.7599 | Regression_cos_phi: 1.8886 | Regression_energy: 1.7189 | Regression_eta: 0.8097 | Regression_pt: 2.0617 | Regression_sin_phi: 2.2480 | LR: 3.47e-04 | DataLoad Time: 0.0012s | Model Forward Time: 3.0311s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.1917 | max_loss: 0.0039 | max_backward: 0.1029 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 5304.8 | uss_MB: 4464.1 | child_rss_MB: 9445.8 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 86.4 +rank3 [2026-10-27 17:39:30,100] INFO : Step 26000/100000 rank3 | Train Loss: 12.6920 | Train Loss Unweighted: 4.5479 | Train Loss Components: Classification: 0.0188 | Classification_binary: 3.6003 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4034 | Regression_eta: 0.0734 | Regression_pt: 0.3564 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.7511 | physical_nopu: 4.6505 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.5548 | Classification_binary: 3.4642 | Regression_cos_phi: 2.1479 | Regression_energy: 2.1281 | Regression_eta: 0.6664 | Regression_pt: 2.8976 | Regression_sin_phi: 1.5785 | LR: 3.43e-04 | DataLoad Time: 0.0472s | Model Forward Time: 3.0487s | Phase Times: max_pre_step_barrier: 0.0479 | max_batch_to_device: 0.0008 | max_forward: 0.1278 | max_loss: 0.0037 | max_backward: 0.0877 | max_optimizer_step: 0.1868 | max_tensorboard_logging: 1.8908 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5292.2 | uss_MB: 4467.2 | child_rss_MB: 9407.7 | allocated_MB: 395.0 | reserved_MB: 12220.0 | inactive_split_MB: 115.0 +rank3 [2026-10-27 17:42:32,922] INFO : Step 27000/100000 rank3 | Train Loss: 16.2758 | Train Loss Unweighted: 5.1421 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5608 | Regression_cos_phi: 0.0448 | Regression_energy: 0.7180 | Regression_eta: 0.0780 | Regression_pt: 0.6791 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.7443 | physical_nopu: 4.6426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3852 | Classification_binary: 3.3745 | Regression_cos_phi: 1.3306 | Regression_energy: 4.8963 | Regression_eta: 0.7170 | Regression_pt: 7.0813 | Regression_sin_phi: 1.4046 | LR: 3.39e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.0077s | Phase Times: max_pre_step_barrier: 0.0385 | max_batch_to_device: 0.0009 | max_forward: 0.0708 | max_loss: 0.0038 | max_backward: 0.0914 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.8878 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5283.2 | uss_MB: 4469.8 | child_rss_MB: 9400.6 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 87.9 +rank3 [2026-10-27 17:45:38,004] INFO : Step 28000/100000 rank3 | Train Loss: 11.9115 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6175 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3339 | Regression_eta: 0.0761 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.7393 | physical_nopu: 4.6288 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2590 | Classification_binary: 3.7597 | Regression_cos_phi: 1.8744 | Regression_energy: 1.8173 | Regression_eta: 0.9158 | Regression_pt: 2.1502 | Regression_sin_phi: 2.0665 | LR: 3.35e-04 | DataLoad Time: 0.0013s | Model Forward Time: 4.8028s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0008 | max_forward: 0.0886 | max_loss: 0.0038 | max_backward: 0.1193 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 3.6285 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 5274.2 | uss_MB: 4146.2 | child_rss_MB: 9355.1 | allocated_MB: 394.0 | reserved_MB: 12414.0 | inactive_split_MB: 106.5 +rank3 [2026-10-27 17:48:54,715] INFO : Step 29000/100000 rank3 | Train Loss: 13.9221 | Train Loss Unweighted: 4.6615 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.5075 | Regression_cos_phi: 0.0491 | Regression_energy: 0.5008 | Regression_eta: 0.0786 | Regression_pt: 0.4589 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7405 | physical_nopu: 4.6233 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3630 | Classification_binary: 3.3374 | Regression_cos_phi: 1.8069 | Regression_energy: 3.4088 | Regression_eta: 0.8361 | Regression_pt: 4.9038 | Regression_sin_phi: 1.6429 | LR: 3.30e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.3814s | Phase Times: max_pre_step_barrier: 0.1677 | max_batch_to_device: 0.0009 | max_forward: 0.0754 | max_loss: 0.0038 | max_backward: 0.0825 | max_optimizer_step: 0.1673 | max_tensorboard_logging: 2.1272 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5252.2 | uss_MB: 4453.2 | child_rss_MB: 9260.7 | allocated_MB: 394.9 | reserved_MB: 12414.0 | inactive_split_MB: 107.1 +rank3 [2026-10-27 17:51:57,834] INFO : Step 30000/100000 rank3 | Train Loss: 12.4988 | Train Loss Unweighted: 4.5639 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.6431 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3981 | Regression_eta: 0.0722 | Regression_pt: 0.3405 | Regression_sin_phi: 0.0498 | Group Train Loss Unweighted (window mean): physical_pu: 3.7342 | physical_nopu: 4.6125 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2669 | Classification_binary: 3.6362 | Regression_cos_phi: 1.4284 | Regression_energy: 2.6370 | Regression_eta: 0.7426 | Regression_pt: 3.4883 | Regression_sin_phi: 1.7067 | LR: 3.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.4591s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0762 | max_loss: 0.0136 | max_backward: 0.0854 | max_optimizer_step: 0.1722 | max_tensorboard_logging: 1.3349 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5242.2 | uss_MB: 4475.0 | child_rss_MB: 9193.8 | allocated_MB: 393.7 | reserved_MB: 12480.0 | inactive_split_MB: 89.7 +rank3 [2026-10-27 17:51:58,780] INFO : Running validation on rank3 +rank3 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 17:53:53,313] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 17:53:53,721] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 17:54:18,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 17:54:18,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 17:54:19,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 17:54:19,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 17:54:20,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 17:54:20,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 17:54:21,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 17:54:21,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 17:54:31,706] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 17:54:31,840] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 17:54:32,213] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 17:54:47,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 17:54:47,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 17:54:48,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 17:54:48,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 17:54:49,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 17:54:49,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 17:54:49,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 17:54:50,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 17:55:00,411] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 17:55:00,637] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 17:55:01,041] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 17:55:16,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 17:55:17,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 17:55:17,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 17:55:18,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 17:55:18,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 17:55:19,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 17:55:19,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 17:55:19,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 17:55:29,946] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-27 17:58:47,285] INFO : Step 31000/100000 rank3 | Train Loss: 12.0027 | Train Loss Unweighted: 4.4503 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.5779 | Regression_cos_phi: 0.0460 | Regression_energy: 0.3731 | Regression_eta: 0.0704 | Regression_pt: 0.3230 | Regression_sin_phi: 0.0451 | Group Train Loss Unweighted (window mean): physical_pu: 3.7262 | physical_nopu: 4.6083 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3493 | Classification_binary: 3.6843 | Regression_cos_phi: 1.9354 | Regression_energy: 1.7304 | Regression_eta: 0.7661 | Regression_pt: 2.2919 | Regression_sin_phi: 1.9554 | LR: 3.21e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.5859s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.1458 | max_loss: 0.0037 | max_backward: 0.1256 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 2.3030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7018.1 | uss_MB: 6249.8 | child_rss_MB: 9197.9 | allocated_MB: 394.8 | reserved_MB: 11168.0 | inactive_split_MB: 96.2 +rank3 [2026-10-27 18:01:54,932] INFO : Step 32000/100000 rank3 | Train Loss: 13.5217 | Train Loss Unweighted: 4.6918 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.5357 | Regression_cos_phi: 0.0443 | Regression_energy: 0.5169 | Regression_eta: 0.0728 | Regression_pt: 0.4649 | Regression_sin_phi: 0.0409 | Group Train Loss Unweighted (window mean): physical_pu: 3.7264 | physical_nopu: 4.6006 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3858 | Classification_binary: 3.7031 | Regression_cos_phi: 1.9857 | Regression_energy: 2.0721 | Regression_eta: 0.7808 | Regression_pt: 2.7338 | Regression_sin_phi: 1.8087 | LR: 3.16e-04 | DataLoad Time: 0.0078s | Model Forward Time: 2.3905s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.0795 | max_loss: 0.0917 | max_backward: 0.1851 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.1890 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5310.0 | uss_MB: 4562.1 | child_rss_MB: 9157.4 | allocated_MB: 394.9 | reserved_MB: 11240.0 | inactive_split_MB: 92.3 +rank3 [2026-10-27 18:04:57,677] INFO : Step 33000/100000 rank3 | Train Loss: 11.0126 | Train Loss Unweighted: 3.7465 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9928 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2905 | Regression_eta: 0.1058 | Regression_pt: 0.2472 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.7276 | physical_nopu: 4.5909 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3451 | Classification_binary: 2.9410 | Regression_cos_phi: 1.6919 | Regression_energy: 1.3029 | Regression_eta: 1.2086 | Regression_pt: 1.6025 | Regression_sin_phi: 1.6293 | LR: 3.12e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.8904s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0027 | max_forward: 0.1127 | max_loss: 0.0150 | max_backward: 0.3396 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.4660 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5295.8 | uss_MB: 4291.2 | child_rss_MB: 9057.9 | allocated_MB: 404.8 | reserved_MB: 11922.0 | inactive_split_MB: 135.4 +rank3 [2026-10-27 18:07:50,121] INFO : Step 34000/100000 rank3 | Train Loss: 14.2400 | Train Loss Unweighted: 4.7634 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5051 | Regression_cos_phi: 0.0450 | Regression_energy: 0.5627 | Regression_eta: 0.0631 | Regression_pt: 0.5251 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7180 | physical_nopu: 4.5858 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3699 | Classification_binary: 3.4818 | Regression_cos_phi: 1.7868 | Regression_energy: 2.9053 | Regression_eta: 0.6257 | Regression_pt: 3.8388 | Regression_sin_phi: 1.2825 | LR: 3.07e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.7410s | Phase Times: max_pre_step_barrier: 0.1802 | max_batch_to_device: 0.0765 | max_forward: 0.1650 | max_loss: 0.0036 | max_backward: 0.0935 | max_optimizer_step: 0.2788 | max_tensorboard_logging: 1.2621 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5285.5 | uss_MB: 4563.5 | child_rss_MB: 9059.3 | allocated_MB: 395.5 | reserved_MB: 11922.0 | inactive_split_MB: 99.3 +rank3 [2026-10-27 18:10:53,637] INFO : Step 35000/100000 rank3 | Train Loss: 11.4005 | Train Loss Unweighted: 4.4142 | Train Loss Components: Classification: 0.0091 | Classification_binary: 3.6972 | Regression_cos_phi: 0.0530 | Regression_energy: 0.2984 | Regression_eta: 0.0671 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.7147 | physical_nopu: 4.5802 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1597 | Classification_binary: 3.4748 | Regression_cos_phi: 1.8902 | Regression_energy: 1.4037 | Regression_eta: 0.6858 | Regression_pt: 1.7301 | Regression_sin_phi: 1.4909 | LR: 3.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.4159s | Phase Times: max_pre_step_barrier: 0.0682 | max_batch_to_device: 0.0008 | max_forward: 0.1243 | max_loss: 0.0038 | max_backward: 0.0883 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.1949 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5274.5 | uss_MB: 4566.4 | child_rss_MB: 9063.3 | allocated_MB: 393.8 | reserved_MB: 11922.0 | inactive_split_MB: 85.2 +rank3 [2026-10-27 18:13:50,577] INFO : Step 36000/100000 rank3 | Train Loss: 11.8438 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.5751 | Regression_cos_phi: 0.0431 | Regression_energy: 0.3555 | Regression_eta: 0.0641 | Regression_pt: 0.3210 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.7110 | physical_nopu: 4.5729 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3507 | Classification_binary: 3.6706 | Regression_cos_phi: 1.3254 | Regression_energy: 1.5833 | Regression_eta: 0.6404 | Regression_pt: 2.1337 | Regression_sin_phi: 1.3747 | LR: 2.97e-04 | DataLoad Time: 0.0007s | Model Forward Time: 2.4524s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0993 | max_loss: 0.0066 | max_backward: 0.1760 | max_optimizer_step: 0.1735 | max_tensorboard_logging: 1.2166 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 5259.5 | uss_MB: 4568.3 | child_rss_MB: 9032.9 | allocated_MB: 395.0 | reserved_MB: 11922.0 | inactive_split_MB: 95.2 +rank3 [2026-10-27 18:16:45,596] INFO : Step 37000/100000 rank3 | Train Loss: 15.1390 | Train Loss Unweighted: 4.8166 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4396 | Regression_cos_phi: 0.0489 | Regression_energy: 0.6142 | Regression_eta: 0.0886 | Regression_pt: 0.5621 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7102 | physical_nopu: 4.5673 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3422 | Classification_binary: 3.4851 | Regression_cos_phi: 2.1290 | Regression_energy: 3.7861 | Regression_eta: 0.9006 | Regression_pt: 5.1216 | Regression_sin_phi: 1.5828 | LR: 2.91e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.2787s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1889 | max_loss: 0.0036 | max_backward: 0.0899 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 1.0901 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 5243.2 | uss_MB: 4569.9 | child_rss_MB: 9022.4 | allocated_MB: 394.3 | reserved_MB: 11922.0 | inactive_split_MB: 107.1 +rank3 [2026-10-27 18:19:42,075] INFO : Step 38000/100000 rank3 | Train Loss: 11.6256 | Train Loss Unweighted: 4.4424 | Train Loss Components: Classification: 0.0117 | Classification_binary: 3.6501 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3369 | Regression_eta: 0.0664 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7131 | physical_nopu: 4.5682 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2367 | Classification_binary: 3.5879 | Regression_cos_phi: 1.6406 | Regression_energy: 1.4674 | Regression_eta: 0.8479 | Regression_pt: 1.7959 | Regression_sin_phi: 1.3714 | LR: 2.86e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.2499s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.0965 | max_loss: 0.0038 | max_backward: 0.1051 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5237.2 | uss_MB: 4571.0 | child_rss_MB: 9030.3 | allocated_MB: 393.3 | reserved_MB: 12082.0 | inactive_split_MB: 110.5 +rank3 [2026-10-27 18:22:37,812] INFO : Step 39000/100000 rank3 | Train Loss: 13.0208 | Train Loss Unweighted: 4.4928 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4758 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4360 | Regression_eta: 0.0793 | Regression_pt: 0.3879 | Regression_sin_phi: 0.0500 | Group Train Loss Unweighted (window mean): physical_pu: 3.7060 | physical_nopu: 4.5607 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3005 | Classification_binary: 3.4320 | Regression_cos_phi: 1.5984 | Regression_energy: 1.4777 | Regression_eta: 0.6914 | Regression_pt: 1.9955 | Regression_sin_phi: 1.3012 | LR: 2.81e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.2314s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.0864 | max_loss: 0.0038 | max_backward: 0.0877 | max_optimizer_step: 0.1700 | max_tensorboard_logging: 1.1418 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5224.7 | uss_MB: 4572.8 | child_rss_MB: 9008.2 | allocated_MB: 394.4 | reserved_MB: 12706.0 | inactive_split_MB: 84.9 +rank3 [2026-10-27 18:25:31,277] INFO : Step 40000/100000 rank3 | Train Loss: 14.7562 | Train Loss Unweighted: 4.9007 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5708 | Regression_cos_phi: 0.0459 | Regression_energy: 0.5954 | Regression_eta: 0.0747 | Regression_pt: 0.5553 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7006 | physical_nopu: 4.5579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2839 | Classification_binary: 3.7188 | Regression_cos_phi: 1.5790 | Regression_energy: 1.5737 | Regression_eta: 0.7309 | Regression_pt: 1.9130 | Regression_sin_phi: 1.6186 | LR: 2.76e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.3072s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.0814 | max_loss: 0.0614 | max_backward: 0.1417 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 1.1378 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5222.7 | uss_MB: 4573.6 | child_rss_MB: 8992.8 | allocated_MB: 393.7 | reserved_MB: 12706.0 | inactive_split_MB: 107.7 +rank3 [2026-10-27 18:25:32,320] INFO : Running validation on rank3 +rank3 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 18:27:23,860] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 18:27:24,220] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 18:27:48,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 18:27:48,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 18:27:49,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 18:27:49,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 18:27:50,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 18:27:50,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 18:27:51,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 18:27:51,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 18:28:01,582] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 18:28:01,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 18:28:02,237] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 18:28:17,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 18:28:17,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 18:28:18,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 18:28:18,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 18:28:19,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 18:28:19,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 18:28:20,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 18:28:20,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 18:28:30,640] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 18:28:30,988] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 18:28:31,367] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 18:28:47,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 18:28:47,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 18:28:47,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 18:28:48,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 18:28:48,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 18:28:49,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 18:28:49,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 18:28:50,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 18:29:00,294] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 18:32:27,442] INFO : Step 41000/100000 rank3 | Train Loss: 12.3609 | Train Loss Unweighted: 4.4817 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5838 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3837 | Regression_eta: 0.0731 | Regression_pt: 0.3287 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.7016 | physical_nopu: 4.5433 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3067 | Classification_binary: 3.5344 | Regression_cos_phi: 1.8536 | Regression_energy: 1.7196 | Regression_eta: 0.7833 | Regression_pt: 2.0414 | Regression_sin_phi: 1.8387 | LR: 2.70e-04 | DataLoad Time: 0.0029s | Model Forward Time: 4.5162s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0009 | max_forward: 0.1338 | max_loss: 0.0038 | max_backward: 0.1857 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 3.1120 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 7028.8 | uss_MB: 6392.1 | child_rss_MB: 8998.3 | allocated_MB: 394.0 | reserved_MB: 11364.0 | inactive_split_MB: 106.1 +rank3 [2026-10-27 18:35:19,857] INFO : Step 42000/100000 rank3 | Train Loss: 12.9790 | Train Loss Unweighted: 4.5354 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4379 | Regression_cos_phi: 0.0458 | Regression_energy: 0.4856 | Regression_eta: 0.0625 | Regression_pt: 0.4500 | Regression_sin_phi: 0.0370 | Group Train Loss Unweighted (window mean): physical_pu: 3.6959 | physical_nopu: 4.5474 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2785 | Classification_binary: 3.3940 | Regression_cos_phi: 1.5207 | Regression_energy: 2.0032 | Regression_eta: 0.4500 | Regression_pt: 2.5732 | Regression_sin_phi: 1.2277 | LR: 2.65e-04 | DataLoad Time: 0.0117s | Model Forward Time: 2.6322s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0009 | max_forward: 0.1375 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 1.3492 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 7016.8 | uss_MB: 5332.4 | child_rss_MB: 8994.4 | allocated_MB: 394.2 | reserved_MB: 12150.0 | inactive_split_MB: 99.3 +rank3 [2026-10-27 18:38:35,925] INFO : Step 43000/100000 rank3 | Train Loss: 10.9006 | Train Loss Unweighted: 3.7057 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.9740 | Regression_cos_phi: 0.0473 | Regression_energy: 0.2786 | Regression_eta: 0.1094 | Regression_pt: 0.2322 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6942 | physical_nopu: 4.5421 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3600 | Classification_binary: 2.9970 | Regression_cos_phi: 1.8368 | Regression_energy: 1.3602 | Regression_eta: 1.3108 | Regression_pt: 1.6542 | Regression_sin_phi: 1.7415 | LR: 2.59e-04 | DataLoad Time: 0.0017s | Model Forward Time: 5.2294s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0039 | max_forward: 0.1100 | max_loss: 0.0637 | max_backward: 0.2303 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 3.9250 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5285.0 | uss_MB: 4686.1 | child_rss_MB: 8875.5 | allocated_MB: 406.8 | reserved_MB: 12150.0 | inactive_split_MB: 152.7 +rank3 [2026-10-27 18:41:39,775] INFO : Step 44000/100000 rank3 | Train Loss: 14.8629 | Train Loss Unweighted: 4.8399 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4856 | Regression_cos_phi: 0.0438 | Regression_energy: 0.6070 | Regression_eta: 0.0739 | Regression_pt: 0.5644 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6933 | physical_nopu: 4.5331 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3805 | Classification_binary: 3.4562 | Regression_cos_phi: 1.9314 | Regression_energy: 3.3562 | Regression_eta: 0.9130 | Regression_pt: 5.3703 | Regression_sin_phi: 1.4742 | LR: 2.54e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.6836s | Phase Times: max_pre_step_barrier: 0.0851 | max_batch_to_device: 0.0008 | max_forward: 0.0946 | max_loss: 0.0041 | max_backward: 0.0964 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.3558 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5284.0 | uss_MB: 4679.1 | child_rss_MB: 8885.5 | allocated_MB: 394.8 | reserved_MB: 12150.0 | inactive_split_MB: 94.9 +rank3 [2026-10-27 18:44:46,984] INFO : Step 45000/100000 rank3 | Train Loss: 12.9535 | Train Loss Unweighted: 4.5658 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.6115 | Regression_cos_phi: 0.0537 | Regression_energy: 0.4117 | Regression_eta: 0.0846 | Regression_pt: 0.3413 | Regression_sin_phi: 0.0531 | Group Train Loss Unweighted (window mean): physical_pu: 3.6892 | physical_nopu: 4.5283 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1839 | Classification_binary: 3.5381 | Regression_cos_phi: 1.4544 | Regression_energy: 1.4669 | Regression_eta: 0.4819 | Regression_pt: 1.9873 | Regression_sin_phi: 1.1713 | LR: 2.48e-04 | DataLoad Time: 0.0028s | Model Forward Time: 2.4260s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1077 | max_loss: 0.0040 | max_backward: 0.1457 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 1.2404 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 5275.4 | uss_MB: 4680.1 | child_rss_MB: 8893.2 | allocated_MB: 394.0 | reserved_MB: 13366.0 | inactive_split_MB: 74.2 +rank3 [2026-10-27 18:48:06,241] INFO : Step 46000/100000 rank3 | Train Loss: 12.3783 | Train Loss Unweighted: 4.4476 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.5252 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3923 | Regression_eta: 0.0679 | Regression_pt: 0.3506 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6869 | physical_nopu: 4.5281 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4978 | Classification_binary: 3.5530 | Regression_cos_phi: 1.3102 | Regression_energy: 1.8134 | Regression_eta: 0.8001 | Regression_pt: 2.3600 | Regression_sin_phi: 1.4021 | LR: 2.43e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.4370s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1021 | max_loss: 0.0036 | max_backward: 0.1895 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 1.1894 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 5269.4 | uss_MB: 4681.7 | child_rss_MB: 8897.5 | allocated_MB: 395.1 | reserved_MB: 13366.0 | inactive_split_MB: 103.1 +rank3 [2026-10-27 18:51:16,142] INFO : Step 47000/100000 rank3 | Train Loss: 14.5142 | Train Loss Unweighted: 4.7631 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.3922 | Regression_cos_phi: 0.0407 | Regression_energy: 0.6220 | Regression_eta: 0.0662 | Regression_pt: 0.5862 | Regression_sin_phi: 0.0406 | Group Train Loss Unweighted (window mean): physical_pu: 3.6853 | physical_nopu: 4.5234 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4087 | Classification_binary: 3.1339 | Regression_cos_phi: 1.4424 | Regression_energy: 3.1948 | Regression_eta: 0.5778 | Regression_pt: 4.5908 | Regression_sin_phi: 1.0324 | LR: 2.37e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.5375s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0667 | max_forward: 0.1426 | max_loss: 0.0037 | max_backward: 0.0873 | max_optimizer_step: 0.2591 | max_tensorboard_logging: 1.1650 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5267.4 | uss_MB: 4682.3 | child_rss_MB: 8896.3 | allocated_MB: 394.9 | reserved_MB: 14166.0 | inactive_split_MB: 95.2 +rank3 [2026-10-27 18:54:12,871] INFO : Step 48000/100000 rank3 | Train Loss: 11.2030 | Train Loss Unweighted: 4.3343 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.6447 | Regression_cos_phi: 0.0521 | Regression_energy: 0.2822 | Regression_eta: 0.0657 | Regression_pt: 0.2293 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6844 | physical_nopu: 4.5168 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2515 | Classification_binary: 3.7632 | Regression_cos_phi: 2.1812 | Regression_energy: 1.5818 | Regression_eta: 0.7465 | Regression_pt: 1.7929 | Regression_sin_phi: 1.7999 | LR: 2.31e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.2608s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.0058 | max_backward: 0.0852 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1613 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5253.4 | uss_MB: 4683.3 | child_rss_MB: 8890.9 | allocated_MB: 393.9 | reserved_MB: 14166.0 | inactive_split_MB: 101.2 +rank3 [2026-10-27 18:57:19,782] INFO : Step 49000/100000 rank3 | Train Loss: 12.4372 | Train Loss Unweighted: 4.4483 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.5320 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3881 | Regression_eta: 0.0742 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6801 | physical_nopu: 4.5174 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3685 | Classification_binary: 3.4514 | Regression_cos_phi: 1.7779 | Regression_energy: 1.9574 | Regression_eta: 0.7052 | Regression_pt: 2.4307 | Regression_sin_phi: 1.6427 | LR: 2.26e-04 | DataLoad Time: 0.0161s | Model Forward Time: 2.4598s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0009 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1014 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 1.3340 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5240.2 | uss_MB: 4683.4 | child_rss_MB: 8815.5 | allocated_MB: 393.7 | reserved_MB: 14166.0 | inactive_split_MB: 100.8 +rank3 [2026-10-27 19:00:32,233] INFO : Step 50000/100000 rank3 | Train Loss: 13.3654 | Train Loss Unweighted: 4.5742 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4424 | Regression_cos_phi: 0.0426 | Regression_energy: 0.5013 | Regression_eta: 0.0683 | Regression_pt: 0.4573 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6814 | physical_nopu: 4.5132 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4249 | Classification_binary: 3.3864 | Regression_cos_phi: 1.6455 | Regression_energy: 2.9732 | Regression_eta: 0.7871 | Regression_pt: 4.0797 | Regression_sin_phi: 1.3637 | LR: 2.20e-04 | DataLoad Time: 0.0015s | Model Forward Time: 2.5864s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0010 | max_forward: 0.1892 | max_loss: 0.0115 | max_backward: 0.1333 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.2641 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0862 | Memory: rss_MB: 5234.2 | uss_MB: 4685.5 | child_rss_MB: 8779.3 | allocated_MB: 394.6 | reserved_MB: 14166.0 | inactive_split_MB: 91.2 +rank3 [2026-10-27 19:00:33,174] INFO : Running validation on rank3 +rank3 [2026-10-27 19:00:33,174] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 19:00:33,174] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 19:00:33,174] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 19:02:37,501] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 19:02:37,940] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 19:03:02,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 19:03:03,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 19:03:03,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 19:03:04,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 19:03:04,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 19:03:05,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 19:03:05,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 19:03:06,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 19:03:16,051] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 19:03:16,262] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 19:03:16,635] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 19:03:31,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 19:03:31,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 19:03:32,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 19:03:32,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 19:03:33,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 19:03:33,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 19:03:34,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 19:03:34,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 19:03:44,728] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 19:03:45,438] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 19:03:45,816] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 19:04:00,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 19:04:01,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 19:04:01,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 19:04:01,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 19:04:02,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 19:04:02,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 19:04:03,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 19:04:03,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 19:04:13,853] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 19:07:44,423] INFO : Step 51000/100000 rank3 | Train Loss: 12.0993 | Train Loss Unweighted: 4.3946 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5485 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3591 | Regression_eta: 0.0789 | Regression_pt: 0.2941 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6745 | physical_nopu: 4.5037 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2732 | Classification_binary: 3.5184 | Regression_cos_phi: 2.8043 | Regression_energy: 2.6270 | Regression_eta: 2.1234 | Regression_pt: 2.8981 | Regression_sin_phi: 2.2012 | LR: 2.14e-04 | DataLoad Time: 0.0098s | Model Forward Time: 3.3976s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0008 | max_forward: 0.1391 | max_loss: 0.0036 | max_backward: 0.1509 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 2.0683 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 7102.6 | uss_MB: 6555.6 | child_rss_MB: 8775.1 | allocated_MB: 393.6 | reserved_MB: 10952.0 | inactive_split_MB: 101.2 +rank3 [2026-10-27 19:10:55,345] INFO : Step 52000/100000 rank3 | Train Loss: 10.7195 | Train Loss Unweighted: 3.6688 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9553 | Regression_cos_phi: 0.0470 | Regression_energy: 0.2719 | Regression_eta: 0.1059 | Regression_pt: 0.2258 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6784 | physical_nopu: 4.5078 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3415 | Classification_binary: 2.9506 | Regression_cos_phi: 1.7537 | Regression_energy: 1.2945 | Regression_eta: 1.1661 | Regression_pt: 1.5871 | Regression_sin_phi: 1.5648 | LR: 2.09e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.5036s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0031 | max_forward: 0.1853 | max_loss: 0.0256 | max_backward: 0.2604 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 2.0239 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7092.6 | uss_MB: 6564.5 | child_rss_MB: 8773.5 | allocated_MB: 404.8 | reserved_MB: 11626.0 | inactive_split_MB: 256.4 +rank3 [2026-10-27 19:13:48,854] INFO : Step 53000/100000 rank3 | Train Loss: 11.7022 | Train Loss Unweighted: 4.3546 | Train Loss Components: Classification: 0.0123 | Classification_binary: 3.5594 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3321 | Regression_eta: 0.0730 | Regression_pt: 0.2795 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6748 | physical_nopu: 4.5005 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3093 | Classification_binary: 3.5187 | Regression_cos_phi: 1.7904 | Regression_energy: 1.7672 | Regression_eta: 0.7622 | Regression_pt: 2.1669 | Regression_sin_phi: 1.4001 | LR: 2.03e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.1499s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0954 | max_loss: 0.0037 | max_backward: 0.1065 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.9892 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 5255.7 | uss_MB: 4729.4 | child_rss_MB: 8773.6 | allocated_MB: 393.6 | reserved_MB: 11910.0 | inactive_split_MB: 100.6 +rank3 [2026-10-27 19:17:18,773] INFO : Step 54000/100000 rank3 | Train Loss: 12.5762 | Train Loss Unweighted: 4.3665 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0487 | Regression_energy: 0.4104 | Regression_eta: 0.0739 | Regression_pt: 0.3685 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6696 | physical_nopu: 4.5036 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3511 | Classification_binary: 3.4287 | Regression_cos_phi: 1.5304 | Regression_energy: 2.6355 | Regression_eta: 0.5377 | Regression_pt: 3.5993 | Regression_sin_phi: 1.3897 | LR: 1.97e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.3956s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.1022 | max_forward: 0.1953 | max_loss: 0.0037 | max_backward: 0.0975 | max_optimizer_step: 0.2772 | max_tensorboard_logging: 2.0527 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5248.7 | uss_MB: 4730.3 | child_rss_MB: 8744.8 | allocated_MB: 394.9 | reserved_MB: 11910.0 | inactive_split_MB: 118.2 +rank3 [2026-10-27 19:20:28,222] INFO : Step 55000/100000 rank3 | Train Loss: 13.4632 | Train Loss Unweighted: 4.6500 | Train Loss Components: Classification: 0.0134 | Classification_binary: 3.5428 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4921 | Regression_eta: 0.0679 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6720 | physical_nopu: 4.4919 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2615 | Classification_binary: 3.5421 | Regression_cos_phi: 2.1287 | Regression_energy: 2.3119 | Regression_eta: 0.6272 | Regression_pt: 2.7584 | Regression_sin_phi: 1.7928 | LR: 1.92e-04 | DataLoad Time: 0.0007s | Model Forward Time: 3.8033s | Phase Times: max_pre_step_barrier: 0.1980 | max_batch_to_device: 0.0007 | max_forward: 0.0889 | max_loss: 0.0037 | max_backward: 0.2091 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 2.3779 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5235.8 | uss_MB: 4729.6 | child_rss_MB: 8640.7 | allocated_MB: 393.8 | reserved_MB: 11910.0 | inactive_split_MB: 89.1 +rank3 [2026-10-27 19:23:30,605] INFO : Step 56000/100000 rank3 | Train Loss: 11.6078 | Train Loss Unweighted: 4.2962 | Train Loss Components: Classification: 0.0140 | Classification_binary: 3.4759 | Regression_cos_phi: 0.0438 | Regression_energy: 0.3484 | Regression_eta: 0.0671 | Regression_pt: 0.2977 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6692 | physical_nopu: 4.4908 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3237 | Classification_binary: 3.4013 | Regression_cos_phi: 1.4632 | Regression_energy: 1.9818 | Regression_eta: 1.1175 | Regression_pt: 2.4564 | Regression_sin_phi: 2.2776 | LR: 1.86e-04 | DataLoad Time: 0.0427s | Model Forward Time: 3.1887s | Phase Times: max_pre_step_barrier: 0.0941 | max_batch_to_device: 0.0008 | max_forward: 0.0930 | max_loss: 0.0038 | max_backward: 0.0988 | max_optimizer_step: 0.1756 | max_tensorboard_logging: 1.8920 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 5221.8 | uss_MB: 4732.9 | child_rss_MB: 8582.9 | allocated_MB: 393.5 | reserved_MB: 11910.0 | inactive_split_MB: 115.7 +rank3 [2026-10-27 19:26:52,902] INFO : Step 57000/100000 rank3 | Train Loss: 13.9252 | Train Loss Unweighted: 4.6468 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4640 | Regression_cos_phi: 0.0464 | Regression_energy: 0.5144 | Regression_eta: 0.0781 | Regression_pt: 0.4791 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6687 | physical_nopu: 4.4875 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3622 | Classification_binary: 3.4525 | Regression_cos_phi: 2.4352 | Regression_energy: 2.2140 | Regression_eta: 1.2005 | Regression_pt: 2.8921 | Regression_sin_phi: 1.3954 | LR: 1.81e-04 | DataLoad Time: 0.0022s | Model Forward Time: 4.6260s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.1223 | max_forward: 0.2094 | max_loss: 0.0420 | max_backward: 0.1628 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 2.9072 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 5216.8 | uss_MB: 4734.2 | child_rss_MB: 8511.2 | allocated_MB: 394.3 | reserved_MB: 12240.0 | inactive_split_MB: 101.4 +rank3 [2026-10-27 19:30:07,651] INFO : Step 58000/100000 rank3 | Train Loss: 11.5910 | Train Loss Unweighted: 4.3047 | Train Loss Components: Classification: 0.0106 | Classification_binary: 3.5490 | Regression_cos_phi: 0.0544 | Regression_energy: 0.3108 | Regression_eta: 0.0631 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0510 | Group Train Loss Unweighted (window mean): physical_pu: 3.6630 | physical_nopu: 4.4842 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2390 | Classification_binary: 3.6148 | Regression_cos_phi: 2.0415 | Regression_energy: 1.4260 | Regression_eta: 0.7701 | Regression_pt: 1.7612 | Regression_sin_phi: 2.1221 | LR: 1.75e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.5120s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.1080 | max_backward: 0.1964 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.2749 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 5209.3 | uss_MB: 4734.4 | child_rss_MB: 8432.6 | allocated_MB: 393.9 | reserved_MB: 12240.0 | inactive_split_MB: 121.9 +rank3 [2026-10-27 19:33:12,149] INFO : Step 59000/100000 rank3 | Train Loss: 12.2218 | Train Loss Unweighted: 4.4115 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5166 | Regression_cos_phi: 0.0458 | Regression_energy: 0.3794 | Regression_eta: 0.0771 | Regression_pt: 0.3280 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6650 | physical_nopu: 4.4788 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3759 | Classification_binary: 3.6764 | Regression_cos_phi: 1.6183 | Regression_energy: 1.7869 | Regression_eta: 0.7908 | Regression_pt: 2.2400 | Regression_sin_phi: 1.9050 | LR: 1.70e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.2204s | Phase Times: max_pre_step_barrier: 0.6889 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0499 | max_backward: 0.1549 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5201.3 | uss_MB: 4735.4 | child_rss_MB: 8387.3 | allocated_MB: 394.0 | reserved_MB: 12240.0 | inactive_split_MB: 135.3 +rank3 [2026-10-27 19:36:26,517] INFO : Step 60000/100000 rank3 | Train Loss: 14.3810 | Train Loss Unweighted: 4.7229 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.4146 | Regression_cos_phi: 0.0449 | Regression_energy: 0.5841 | Regression_eta: 0.0716 | Regression_pt: 0.5495 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.6604 | physical_nopu: 4.4865 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2683 | Classification_binary: 3.4962 | Regression_cos_phi: 1.5216 | Regression_energy: 2.2504 | Regression_eta: 0.8857 | Regression_pt: 2.9076 | Regression_sin_phi: 1.2678 | LR: 1.64e-04 | DataLoad Time: 0.0031s | Model Forward Time: 2.4559s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0849 | max_loss: 0.0036 | max_backward: 0.1478 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 1.2880 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5191.9 | uss_MB: 4495.0 | child_rss_MB: 8328.9 | allocated_MB: 394.8 | reserved_MB: 12240.0 | inactive_split_MB: 87.0 +rank3 [2026-10-27 19:36:27,558] INFO : Running validation on rank3 +rank3 [2026-10-27 19:36:27,558] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 19:38:18,600] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 19:38:19,028] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 19:38:43,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 19:38:44,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 19:38:44,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 19:38:45,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 19:38:45,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 19:38:45,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 19:38:46,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 19:38:46,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 19:38:56,970] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 19:38:57,241] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 19:38:57,611] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 19:39:12,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 19:39:13,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 19:39:13,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 19:39:14,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 19:39:14,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 19:39:15,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 19:39:15,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 19:39:16,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 19:39:26,163] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 19:39:26,170] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 19:39:26,533] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 19:39:41,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 19:39:41,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 19:39:42,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 19:39:42,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 19:39:43,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 19:39:43,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 19:39:44,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 19:39:44,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 19:39:54,718] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 19:43:33,543] INFO : Step 61000/100000 rank3 | Train Loss: 11.4262 | Train Loss Unweighted: 4.3706 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.6500 | Regression_cos_phi: 0.0531 | Regression_energy: 0.2956 | Regression_eta: 0.0675 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6553 | physical_nopu: 4.4780 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2242 | Classification_binary: 3.8879 | Regression_cos_phi: 1.8667 | Regression_energy: 1.3200 | Regression_eta: 1.0178 | Regression_pt: 1.6517 | Regression_sin_phi: 1.6523 | LR: 1.59e-04 | DataLoad Time: 0.0076s | Model Forward Time: 3.1988s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1116 | max_loss: 0.0036 | max_backward: 0.1506 | max_optimizer_step: 0.1828 | max_tensorboard_logging: 1.9738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 5784.0 | uss_MB: 5329.2 | child_rss_MB: 8302.4 | allocated_MB: 393.5 | reserved_MB: 11190.0 | inactive_split_MB: 103.1 +rank3 [2026-10-27 19:46:49,589] INFO : Step 62000/100000 rank3 | Train Loss: 10.7781 | Train Loss Unweighted: 3.6510 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9253 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2791 | Regression_eta: 0.1043 | Regression_pt: 0.2316 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3733 | Classification_binary: 2.8695 | Regression_cos_phi: 1.8059 | Regression_energy: 1.4079 | Regression_eta: 1.1541 | Regression_pt: 1.7435 | Regression_sin_phi: 1.6730 | LR: 1.54e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.6556s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0030 | max_forward: 0.1294 | max_loss: 0.0038 | max_backward: 0.2007 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.2115 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0496 | Memory: rss_MB: 5775.0 | uss_MB: 5052.2 | child_rss_MB: 8186.6 | allocated_MB: 404.4 | reserved_MB: 11190.0 | inactive_split_MB: 148.6 +rank3 [2026-10-27 19:50:02,443] INFO : Step 63000/100000 rank3 | Train Loss: 11.5766 | Train Loss Unweighted: 4.3655 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6000 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3204 | Regression_eta: 0.0727 | Regression_pt: 0.2610 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6589 | physical_nopu: 4.4771 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2961 | Classification_binary: 3.4004 | Regression_cos_phi: 2.0555 | Regression_energy: 1.3736 | Regression_eta: 0.6938 | Regression_pt: 1.8608 | Regression_sin_phi: 1.4456 | LR: 1.49e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.3896s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.1036 | max_loss: 0.0037 | max_backward: 0.1140 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2208 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 5773.0 | uss_MB: 5331.6 | child_rss_MB: 8138.8 | allocated_MB: 393.8 | reserved_MB: 11190.0 | inactive_split_MB: 93.0 +rank3 [2026-10-27 19:53:17,771] INFO : Step 64000/100000 rank3 | Train Loss: 13.2709 | Train Loss Unweighted: 4.5542 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.4295 | Regression_cos_phi: 0.0429 | Regression_energy: 0.4948 | Regression_eta: 0.0672 | Regression_pt: 0.4587 | Regression_sin_phi: 0.0436 | Group Train Loss Unweighted (window mean): physical_pu: 3.6551 | physical_nopu: 4.4714 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3790 | Classification_binary: 3.6512 | Regression_cos_phi: 1.6087 | Regression_energy: 1.5224 | Regression_eta: 0.8542 | Regression_pt: 1.9277 | Regression_sin_phi: 1.9071 | LR: 1.43e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.3958s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0928 | max_loss: 0.0570 | max_backward: 0.1589 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 1.2035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 5289.6 | uss_MB: 4857.6 | child_rss_MB: 8120.4 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank3 [2026-10-27 19:56:23,053] INFO : Step 65000/100000 rank3 | Train Loss: 15.6652 | Train Loss Unweighted: 4.9833 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5112 | Regression_cos_phi: 0.0496 | Regression_energy: 0.6688 | Regression_eta: 0.0629 | Regression_pt: 0.6290 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6554 | physical_nopu: 4.4650 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3230 | Classification_binary: 3.5276 | Regression_cos_phi: 1.4950 | Regression_energy: 2.4335 | Regression_eta: 0.6943 | Regression_pt: 3.4610 | Regression_sin_phi: 1.5829 | LR: 1.38e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.7632s | Phase Times: max_pre_step_barrier: 0.1158 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0042 | max_backward: 0.1974 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.4176 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5282.6 | uss_MB: 4858.3 | child_rss_MB: 8025.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 124.9 +rank3 [2026-10-27 19:59:28,722] INFO : Step 66000/100000 rank3 | Train Loss: 12.0073 | Train Loss Unweighted: 4.3350 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4949 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3535 | Regression_eta: 0.0691 | Regression_pt: 0.3018 | Regression_sin_phi: 0.0501 | Group Train Loss Unweighted (window mean): physical_pu: 3.6526 | physical_nopu: 4.4657 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3071 | Classification_binary: 3.3758 | Regression_cos_phi: 1.6650 | Regression_energy: 1.3359 | Regression_eta: 0.5729 | Regression_pt: 1.6378 | Regression_sin_phi: 1.2190 | LR: 1.33e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.4475s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0891 | max_loss: 0.0538 | max_backward: 0.1468 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 1.2753 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5277.7 | uss_MB: 4858.9 | child_rss_MB: 7982.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 107.1 +rank3 [2026-10-27 20:02:51,910] INFO : Step 67000/100000 rank3 | Train Loss: 13.3177 | Train Loss Unweighted: 4.5554 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.4800 | Regression_cos_phi: 0.0500 | Regression_energy: 0.4690 | Regression_eta: 0.0673 | Regression_pt: 0.4253 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6538 | physical_nopu: 4.4665 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3126 | Classification_binary: 3.6988 | Regression_cos_phi: 2.1958 | Regression_energy: 2.2479 | Regression_eta: 0.8686 | Regression_pt: 2.8732 | Regression_sin_phi: 1.5179 | LR: 1.28e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.9382s | Phase Times: max_pre_step_barrier: 0.0792 | max_batch_to_device: 0.0009 | max_forward: 0.0886 | max_loss: 0.1057 | max_backward: 0.1987 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 2.6109 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 5271.7 | uss_MB: 4859.8 | child_rss_MB: 7952.8 | allocated_MB: 393.9 | reserved_MB: 11792.0 | inactive_split_MB: 111.5 +rank3 [2026-10-27 20:06:09,103] INFO : Step 68000/100000 rank3 | Train Loss: 11.1535 | Train Loss Unweighted: 4.2643 | Train Loss Components: Classification: 0.0109 | Classification_binary: 3.5138 | Regression_cos_phi: 0.0456 | Regression_energy: 0.3163 | Regression_eta: 0.0648 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6527 | physical_nopu: 4.4693 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2426 | Classification_binary: 3.6376 | Regression_cos_phi: 1.5122 | Regression_energy: 1.4269 | Regression_eta: 0.6687 | Regression_pt: 1.7495 | Regression_sin_phi: 1.4118 | LR: 1.24e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.3901s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1008 | max_loss: 0.0767 | max_backward: 0.1768 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.1759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5263.4 | uss_MB: 4860.3 | child_rss_MB: 7883.7 | allocated_MB: 394.5 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank3 [2026-10-27 20:09:12,492] INFO : Step 69000/100000 rank3 | Train Loss: 12.1728 | Train Loss Unweighted: 4.3418 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4719 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3660 | Regression_eta: 0.0763 | Regression_pt: 0.3115 | Regression_sin_phi: 0.0548 | Group Train Loss Unweighted (window mean): physical_pu: 3.6494 | physical_nopu: 4.4629 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3744 | Classification_binary: 3.5543 | Regression_cos_phi: 1.4964 | Regression_energy: 1.6867 | Regression_eta: 1.0649 | Regression_pt: 2.1326 | Regression_sin_phi: 2.1057 | LR: 1.19e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.2952s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.1094 | max_loss: 0.0147 | max_backward: 0.1167 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 1.1204 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5251.4 | uss_MB: 4861.5 | child_rss_MB: 7816.8 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 106.5 +rank3 [2026-10-27 20:12:13,498] INFO : Step 70000/100000 rank3 | Train Loss: 14.9156 | Train Loss Unweighted: 4.7730 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.3975 | Regression_cos_phi: 0.0487 | Regression_energy: 0.6183 | Regression_eta: 0.0638 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0448 | Group Train Loss Unweighted (window mean): physical_pu: 3.6488 | physical_nopu: 4.4584 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4457 | Classification_binary: 3.3799 | Regression_cos_phi: 1.6268 | Regression_energy: 4.5955 | Regression_eta: 0.8089 | Regression_pt: 6.3682 | Regression_sin_phi: 1.2956 | LR: 1.14e-04 | DataLoad Time: 0.0073s | Model Forward Time: 2.3863s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1038 | max_loss: 0.0858 | max_backward: 0.1871 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 1.1619 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5245.8 | uss_MB: 4860.8 | child_rss_MB: 7775.8 | allocated_MB: 395.0 | reserved_MB: 11792.0 | inactive_split_MB: 111.9 +rank3 [2026-10-27 20:12:14,504] INFO : Running validation on rank3 +rank3 [2026-10-27 20:12:14,504] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 20:14:09,433] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 20:14:09,864] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 20:14:34,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 20:14:35,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 20:14:35,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 20:14:36,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 20:14:36,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 20:14:36,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 20:14:37,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 20:14:37,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 20:14:47,854] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 20:14:47,942] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 20:14:48,315] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 20:15:03,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 20:15:03,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 20:15:04,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 20:15:04,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 20:15:04,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 20:15:05,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 20:15:05,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 20:15:06,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 20:15:16,406] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 20:15:17,038] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 20:15:17,439] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 20:15:32,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 20:15:33,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 20:15:33,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 20:15:33,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 20:15:34,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 20:15:34,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 20:15:35,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 20:15:35,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 20:15:45,842] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 20:19:21,751] INFO : Step 71000/100000 rank3 | Train Loss: 10.9922 | Train Loss Unweighted: 3.6772 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9201 | Regression_cos_phi: 0.0477 | Regression_energy: 0.2906 | Regression_eta: 0.1066 | Regression_pt: 0.2485 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6484 | physical_nopu: 4.4561 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3517 | Classification_binary: 2.9013 | Regression_cos_phi: 1.6907 | Regression_energy: 1.3184 | Regression_eta: 1.2327 | Regression_pt: 1.5893 | Regression_sin_phi: 1.5671 | LR: 1.10e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.5363s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0652 | max_forward: 0.2596 | max_loss: 0.0036 | max_backward: 0.3497 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 1.8772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 7279.8 | uss_MB: 6900.2 | child_rss_MB: 7804.4 | allocated_MB: 405.9 | reserved_MB: 11130.0 | inactive_split_MB: 135.5 +rank3 [2026-10-27 20:22:34,777] INFO : Step 72000/100000 rank3 | Train Loss: 14.4733 | Train Loss Unweighted: 4.6898 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3667 | Regression_cos_phi: 0.0458 | Regression_energy: 0.5948 | Regression_eta: 0.0650 | Regression_pt: 0.5567 | Regression_sin_phi: 0.0444 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4536 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3902 | Classification_binary: 3.3894 | Regression_cos_phi: 1.3963 | Regression_energy: 3.1780 | Regression_eta: 0.7089 | Regression_pt: 4.4496 | Regression_sin_phi: 1.2644 | LR: 1.05e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.5069s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1390 | max_loss: 0.0037 | max_backward: 0.1007 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.2476 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1281 | Memory: rss_MB: 5316.2 | uss_MB: 4936.1 | child_rss_MB: 7727.0 | allocated_MB: 393.8 | reserved_MB: 11706.0 | inactive_split_MB: 88.5 +rank3 [2026-10-27 20:25:34,428] INFO : Step 73000/100000 rank3 | Train Loss: 11.7903 | Train Loss Unweighted: 4.2561 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.5010 | Regression_cos_phi: 0.0578 | Regression_energy: 0.3092 | Regression_eta: 0.0644 | Regression_pt: 0.2565 | Regression_sin_phi: 0.0557 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4567 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2774 | Classification_binary: 3.3611 | Regression_cos_phi: 2.4472 | Regression_energy: 1.6583 | Regression_eta: 0.9108 | Regression_pt: 2.0562 | Regression_sin_phi: 1.8201 | LR: 1.01e-04 | DataLoad Time: 0.0027s | Model Forward Time: 2.6059s | Phase Times: max_pre_step_barrier: 0.1246 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0122 | max_backward: 0.1972 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.2348 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5307.2 | uss_MB: 4937.3 | child_rss_MB: 7690.0 | allocated_MB: 393.5 | reserved_MB: 11706.0 | inactive_split_MB: 93.2 +rank3 [2026-10-27 20:28:47,370] INFO : Step 74000/100000 rank3 | Train Loss: 11.6748 | Train Loss Unweighted: 4.2803 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4309 | Regression_cos_phi: 0.0428 | Regression_energy: 0.3572 | Regression_eta: 0.0695 | Regression_pt: 0.3177 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.6485 | physical_nopu: 4.4502 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3237 | Classification_binary: 3.3225 | Regression_cos_phi: 1.4621 | Regression_energy: 1.6877 | Regression_eta: 0.6751 | Regression_pt: 2.1417 | Regression_sin_phi: 1.5799 | LR: 9.68e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.0483s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1195 | max_loss: 0.0060 | max_backward: 0.2082 | max_optimizer_step: 0.1763 | max_tensorboard_logging: 1.8023 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5300.9 | uss_MB: 4938.3 | child_rss_MB: 7614.5 | allocated_MB: 394.3 | reserved_MB: 11706.0 | inactive_split_MB: 97.0 +rank3 [2026-10-27 20:31:56,443] INFO : Step 75000/100000 rank3 | Train Loss: 14.1604 | Train Loss Unweighted: 4.6445 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.4007 | Regression_cos_phi: 0.0489 | Regression_energy: 0.5577 | Regression_eta: 0.0640 | Regression_pt: 0.5112 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.6489 | physical_nopu: 4.4484 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2511 | Classification_binary: 3.5894 | Regression_cos_phi: 1.7674 | Regression_energy: 1.7618 | Regression_eta: 0.7598 | Regression_pt: 2.0246 | Regression_sin_phi: 1.7274 | LR: 9.27e-05 | DataLoad Time: 0.0021s | Model Forward Time: 2.9330s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0831 | max_loss: 0.0038 | max_backward: 0.2058 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.5751 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1279 | Memory: rss_MB: 5290.9 | uss_MB: 4938.0 | child_rss_MB: 7581.3 | allocated_MB: 394.4 | reserved_MB: 11706.0 | inactive_split_MB: 89.1 +rank3 [2026-10-27 20:34:57,733] INFO : Step 76000/100000 rank3 | Train Loss: 11.4149 | Train Loss Unweighted: 4.2984 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5224 | Regression_cos_phi: 0.0478 | Regression_energy: 0.3298 | Regression_eta: 0.0709 | Regression_pt: 0.2685 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6454 | physical_nopu: 4.4486 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2977 | Classification_binary: 3.5775 | Regression_cos_phi: 1.7846 | Regression_energy: 2.0434 | Regression_eta: 0.8339 | Regression_pt: 2.1449 | Regression_sin_phi: 1.6798 | LR: 8.88e-05 | DataLoad Time: 0.0290s | Model Forward Time: 3.1454s | Phase Times: max_pre_step_barrier: 0.0296 | max_batch_to_device: 0.0996 | max_forward: 0.1816 | max_loss: 0.0036 | max_backward: 0.0930 | max_optimizer_step: 0.2829 | max_tensorboard_logging: 1.7104 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1287 | Memory: rss_MB: 5286.0 | uss_MB: 4937.3 | child_rss_MB: 7586.8 | allocated_MB: 393.6 | reserved_MB: 12330.0 | inactive_split_MB: 97.7 +rank3 [2026-10-27 20:38:16,884] INFO : Step 77000/100000 rank3 | Train Loss: 12.8789 | Train Loss Unweighted: 4.4653 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4333 | Regression_cos_phi: 0.0478 | Regression_energy: 0.4517 | Regression_eta: 0.0624 | Regression_pt: 0.4079 | Regression_sin_phi: 0.0461 | Group Train Loss Unweighted (window mean): physical_pu: 3.6433 | physical_nopu: 4.4448 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3944 | Classification_binary: 3.4844 | Regression_cos_phi: 1.5560 | Regression_energy: 2.4586 | Regression_eta: 0.5981 | Regression_pt: 3.3303 | Regression_sin_phi: 1.7123 | LR: 8.50e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.0366s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.0831 | max_loss: 0.0514 | max_backward: 0.1524 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.8652 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5273.0 | uss_MB: 4939.8 | child_rss_MB: 7463.9 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 99.2 +rank3 [2026-10-27 20:41:21,661] INFO : Step 78000/100000 rank3 | Train Loss: 13.2420 | Train Loss Unweighted: 4.6345 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.5815 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4598 | Regression_eta: 0.0679 | Regression_pt: 0.4144 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6415 | physical_nopu: 4.4494 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3173 | Classification_binary: 3.5155 | Regression_cos_phi: 2.0068 | Regression_energy: 3.0221 | Regression_eta: 0.5647 | Regression_pt: 4.2511 | Regression_sin_phi: 1.4445 | LR: 8.13e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.0718s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0927 | max_loss: 0.0036 | max_backward: 0.2060 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.8450 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5261.0 | uss_MB: 4940.8 | child_rss_MB: 7383.4 | allocated_MB: 394.7 | reserved_MB: 12330.0 | inactive_split_MB: 95.1 +rank3 [2026-10-27 20:44:31,405] INFO : Step 79000/100000 rank3 | Train Loss: 11.5047 | Train Loss Unweighted: 4.2761 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4919 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3261 | Regression_eta: 0.0640 | Regression_pt: 0.2834 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6416 | physical_nopu: 4.4460 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3523 | Classification_binary: 3.4495 | Regression_cos_phi: 1.5544 | Regression_energy: 1.2680 | Regression_eta: 0.7502 | Regression_pt: 1.6083 | Regression_sin_phi: 1.3840 | LR: 7.78e-05 | DataLoad Time: 0.0025s | Model Forward Time: 3.2157s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0253 | max_forward: 0.1473 | max_loss: 0.0038 | max_backward: 0.0955 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 1.9569 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5248.8 | uss_MB: 4513.6 | child_rss_MB: 7321.5 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 120.0 +rank3 [2026-10-27 20:47:43,818] INFO : Step 80000/100000 rank3 | Train Loss: 15.3065 | Train Loss Unweighted: 4.8564 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4208 | Regression_cos_phi: 0.0455 | Regression_energy: 0.6415 | Regression_eta: 0.0773 | Regression_pt: 0.6092 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6398 | physical_nopu: 4.4406 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3833 | Classification_binary: 3.3765 | Regression_cos_phi: 1.3317 | Regression_energy: 3.2934 | Regression_eta: 0.9132 | Regression_pt: 4.3622 | Regression_sin_phi: 1.7738 | LR: 7.44e-05 | DataLoad Time: 0.0088s | Model Forward Time: 3.0636s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0010 | max_forward: 0.0840 | max_loss: 0.0039 | max_backward: 0.0916 | max_optimizer_step: 0.2630 | max_tensorboard_logging: 1.7339 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5238.8 | uss_MB: 4942.0 | child_rss_MB: 7231.1 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 141.7 +rank3 [2026-10-27 20:47:44,748] INFO : Running validation on rank3 +rank3 [2026-10-27 20:47:44,748] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 20:47:44,748] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 20:49:38,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 20:49:38,981] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 20:50:04,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 20:50:05,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 20:50:05,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 20:50:06,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 20:50:06,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 20:50:07,240] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 20:50:07,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 20:50:08,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 20:50:18,197] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 20:50:18,317] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 20:50:18,743] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 20:50:33,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 20:50:34,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 20:50:34,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 20:50:35,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 20:50:35,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 20:50:36,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 20:50:36,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 20:50:37,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 20:50:47,166] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 20:50:47,267] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 20:50:47,677] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 20:51:03,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 20:51:03,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 20:51:03,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 20:51:04,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 20:51:04,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 20:51:05,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 20:51:05,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 20:51:06,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 20:51:16,263] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 20:54:47,405] INFO : Step 81000/100000 rank3 | Train Loss: 10.7072 | Train Loss Unweighted: 3.6241 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.9080 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2710 | Regression_eta: 0.1078 | Regression_pt: 0.2269 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6413 | physical_nopu: 4.4395 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3497 | Classification_binary: 2.8323 | Regression_cos_phi: 1.6185 | Regression_energy: 1.3252 | Regression_eta: 1.2279 | Regression_pt: 1.6270 | Regression_sin_phi: 1.5698 | LR: 7.11e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.4725s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0033 | max_forward: 0.1984 | max_loss: 0.0038 | max_backward: 0.3244 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 1.8991 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 7314.5 | uss_MB: 7027.3 | child_rss_MB: 7215.1 | allocated_MB: 408.0 | reserved_MB: 11650.0 | inactive_split_MB: 261.6 +rank3 [2026-10-27 20:58:04,709] INFO : Step 82000/100000 rank3 | Train Loss: 12.9094 | Train Loss Unweighted: 4.6043 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5247 | Regression_cos_phi: 0.0417 | Regression_energy: 0.4765 | Regression_eta: 0.0603 | Regression_pt: 0.4449 | Regression_sin_phi: 0.0402 | Group Train Loss Unweighted (window mean): physical_pu: 3.6409 | physical_nopu: 4.4420 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3678 | Classification_binary: 3.4260 | Regression_cos_phi: 1.1017 | Regression_energy: 2.1297 | Regression_eta: 0.5711 | Regression_pt: 2.9758 | Regression_sin_phi: 1.1618 | LR: 6.80e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.1713s | Phase Times: max_pre_step_barrier: 0.0156 | max_batch_to_device: 0.0008 | max_forward: 0.1472 | max_loss: 0.0038 | max_backward: 0.1553 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 1.8498 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5342.3 | uss_MB: 5059.9 | child_rss_MB: 7159.0 | allocated_MB: 393.8 | reserved_MB: 11650.0 | inactive_split_MB: 92.9 +rank3 [2026-10-27 21:01:20,554] INFO : Step 83000/100000 rank3 | Train Loss: 11.2556 | Train Loss Unweighted: 4.2618 | Train Loss Components: Classification: 0.0094 | Classification_binary: 3.5167 | Regression_cos_phi: 0.0501 | Regression_energy: 0.3127 | Regression_eta: 0.0669 | Regression_pt: 0.2586 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6408 | physical_nopu: 4.4397 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2134 | Classification_binary: 3.6906 | Regression_cos_phi: 2.1955 | Regression_energy: 1.1793 | Regression_eta: 0.7978 | Regression_pt: 1.4178 | Regression_sin_phi: 1.5635 | LR: 6.51e-05 | DataLoad Time: 0.0036s | Model Forward Time: 2.5734s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0008 | max_forward: 0.1683 | max_loss: 0.0037 | max_backward: 0.1023 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.3519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 5336.3 | uss_MB: 5061.6 | child_rss_MB: 7095.7 | allocated_MB: 393.2 | reserved_MB: 11650.0 | inactive_split_MB: 95.2 +rank3 [2026-10-27 21:04:33,664] INFO : Step 84000/100000 rank3 | Train Loss: 11.5964 | Train Loss Unweighted: 4.2611 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4346 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3486 | Regression_eta: 0.0638 | Regression_pt: 0.3055 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6366 | physical_nopu: 4.4419 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3949 | Classification_binary: 3.5988 | Regression_cos_phi: 1.9490 | Regression_energy: 1.9568 | Regression_eta: 1.0025 | Regression_pt: 2.4130 | Regression_sin_phi: 2.1452 | LR: 6.23e-05 | DataLoad Time: 0.0990s | Model Forward Time: 4.8366s | Phase Times: max_pre_step_barrier: 0.2933 | max_batch_to_device: 0.0008 | max_forward: 0.1129 | max_loss: 0.0110 | max_backward: 0.1312 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 3.4426 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5330.3 | uss_MB: 5063.1 | child_rss_MB: 7029.9 | allocated_MB: 393.9 | reserved_MB: 11650.0 | inactive_split_MB: 95.6 +rank3 [2026-10-27 21:07:28,408] INFO : Step 85000/100000 rank3 | Train Loss: 14.1090 | Train Loss Unweighted: 4.6641 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.3775 | Regression_cos_phi: 0.0425 | Regression_energy: 0.5754 | Regression_eta: 0.0615 | Regression_pt: 0.5479 | Regression_sin_phi: 0.0420 | Group Train Loss Unweighted (window mean): physical_pu: 3.6325 | physical_nopu: 4.4336 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3439 | Classification_binary: 3.3204 | Regression_cos_phi: 1.2896 | Regression_energy: 3.0759 | Regression_eta: 0.6171 | Regression_pt: 4.1719 | Regression_sin_phi: 1.3840 | LR: 5.96e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.5393s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0496 | max_backward: 0.1488 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 1.3583 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 5323.1 | uss_MB: 5063.4 | child_rss_MB: 7037.3 | allocated_MB: 394.0 | reserved_MB: 11698.0 | inactive_split_MB: 93.1 +rank3 [2026-10-27 21:10:26,799] INFO : Step 86000/100000 rank3 | Train Loss: 11.7921 | Train Loss Unweighted: 4.3412 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.5823 | Regression_cos_phi: 0.0537 | Regression_energy: 0.3118 | Regression_eta: 0.0750 | Regression_pt: 0.2521 | Regression_sin_phi: 0.0550 | Group Train Loss Unweighted (window mean): physical_pu: 3.6404 | physical_nopu: 4.4337 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2524 | Classification_binary: 3.5455 | Regression_cos_phi: 2.5104 | Regression_energy: 1.3884 | Regression_eta: 0.7210 | Regression_pt: 1.5334 | Regression_sin_phi: 1.7368 | LR: 5.71e-05 | DataLoad Time: 0.0020s | Model Forward Time: 2.4342s | Phase Times: max_pre_step_barrier: 0.0233 | max_batch_to_device: 0.0898 | max_forward: 0.1824 | max_loss: 0.0038 | max_backward: 0.0938 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.9995 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1323 | Memory: rss_MB: 5319.1 | uss_MB: 4616.7 | child_rss_MB: 7006.7 | allocated_MB: 393.2 | reserved_MB: 11704.0 | inactive_split_MB: 96.0 +rank3 [2026-10-27 21:13:31,178] INFO : Step 87000/100000 rank3 | Train Loss: 12.0054 | Train Loss Unweighted: 4.3232 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.3964 | Regression_cos_phi: 0.0444 | Regression_energy: 0.4046 | Regression_eta: 0.0620 | Regression_pt: 0.3567 | Regression_sin_phi: 0.0418 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3966 | Classification_binary: 3.3346 | Regression_cos_phi: 1.1730 | Regression_energy: 1.5831 | Regression_eta: 0.6158 | Regression_pt: 2.0454 | Regression_sin_phi: 1.3168 | LR: 5.48e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.3325s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.1290 | max_loss: 0.0039 | max_backward: 0.0975 | max_optimizer_step: 0.1784 | max_tensorboard_logging: 1.1620 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5310.9 | uss_MB: 5064.7 | child_rss_MB: 7011.5 | allocated_MB: 394.7 | reserved_MB: 11704.0 | inactive_split_MB: 94.3 +rank3 [2026-10-27 21:16:33,434] INFO : Step 88000/100000 rank3 | Train Loss: 14.2663 | Train Loss Unweighted: 4.6673 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.4533 | Regression_cos_phi: 0.0523 | Regression_energy: 0.5342 | Regression_eta: 0.0666 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0508 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3710 | Classification_binary: 3.3202 | Regression_cos_phi: 1.8040 | Regression_energy: 3.0801 | Regression_eta: 0.7143 | Regression_pt: 4.3312 | Regression_sin_phi: 2.7391 | LR: 5.26e-05 | DataLoad Time: 0.0025s | Model Forward Time: 2.3215s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0008 | max_forward: 0.0997 | max_loss: 0.0039 | max_backward: 0.1038 | max_optimizer_step: 0.2517 | max_tensorboard_logging: 1.0175 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5309.9 | uss_MB: 5065.2 | child_rss_MB: 7012.4 | allocated_MB: 393.7 | reserved_MB: 11704.0 | inactive_split_MB: 103.2 +rank3 [2026-10-27 21:19:30,808] INFO : Step 89000/100000 rank3 | Train Loss: 11.9885 | Train Loss Unweighted: 4.3122 | Train Loss Components: Classification: 0.0138 | Classification_binary: 3.4837 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3438 | Regression_eta: 0.0687 | Regression_pt: 0.2984 | Regression_sin_phi: 0.0554 | Group Train Loss Unweighted (window mean): physical_pu: 3.6329 | physical_nopu: 4.4374 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2945 | Classification_binary: 3.4758 | Regression_cos_phi: 1.4859 | Regression_energy: 1.8083 | Regression_eta: 0.7284 | Regression_pt: 2.2132 | Regression_sin_phi: 1.8127 | LR: 5.06e-05 | DataLoad Time: 0.0024s | Model Forward Time: 2.2333s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0976 | max_loss: 0.0053 | max_backward: 0.1242 | max_optimizer_step: 0.1785 | max_tensorboard_logging: 1.0326 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 5307.5 | uss_MB: 5066.4 | child_rss_MB: 7006.7 | allocated_MB: 395.0 | reserved_MB: 13422.0 | inactive_split_MB: 109.9 +rank3 [2026-10-27 21:22:22,522] INFO : Step 90000/100000 rank3 | Train Loss: 10.5888 | Train Loss Unweighted: 3.6256 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9282 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2656 | Regression_eta: 0.1044 | Regression_pt: 0.2178 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6354 | physical_nopu: 4.4354 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3183 | Classification_binary: 2.8811 | Regression_cos_phi: 1.6953 | Regression_energy: 1.2321 | Regression_eta: 1.1434 | Regression_pt: 1.4712 | Regression_sin_phi: 1.6414 | LR: 4.88e-05 | DataLoad Time: 0.0020s | Model Forward Time: 4.1185s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0033 | max_forward: 0.1953 | max_loss: 0.0645 | max_backward: 0.8258 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.9975 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5307.5 | uss_MB: 5074.7 | child_rss_MB: 7021.2 | allocated_MB: 404.6 | reserved_MB: 13422.0 | inactive_split_MB: 208.4 +rank3 [2026-10-27 21:22:23,503] INFO : Running validation on rank3 +rank3 [2026-10-27 21:22:23,503] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 21:22:23,503] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 21:22:23,504] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 21:22:23,504] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 21:24:23,626] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 21:24:24,116] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 21:24:48,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 21:24:49,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 21:24:49,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 21:24:50,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 21:24:50,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 21:24:51,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 21:24:51,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 21:24:52,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 21:25:02,120] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 21:25:02,472] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 21:25:02,869] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 21:25:18,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 21:25:18,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 21:25:19,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 21:25:19,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 21:25:20,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 21:25:20,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 21:25:21,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 21:25:21,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 21:25:31,581] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 21:25:32,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 21:25:32,493] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 21:25:47,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 21:25:47,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 21:25:48,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 21:25:48,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 21:25:49,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 21:25:49,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 21:25:50,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 21:25:51,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 21:26:01,026] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 21:29:48,334] INFO : Step 91000/100000 rank3 | Train Loss: 11.9808 | Train Loss Unweighted: 4.3520 | Train Loss Components: Classification: 0.0128 | Classification_binary: 3.5260 | Regression_cos_phi: 0.0522 | Regression_energy: 0.3480 | Regression_eta: 0.0723 | Regression_pt: 0.2901 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4346 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3058 | Classification_binary: 3.5954 | Regression_cos_phi: 2.0462 | Regression_energy: 1.7275 | Regression_eta: 0.9927 | Regression_pt: 1.9697 | Regression_sin_phi: 1.9546 | LR: 4.71e-05 | DataLoad Time: 0.0026s | Model Forward Time: 3.6008s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1743 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.3267 | max_tensorboard_logging: 2.0733 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0716 | Memory: rss_MB: 7308.2 | uss_MB: 7066.1 | child_rss_MB: 7044.9 | allocated_MB: 393.7 | reserved_MB: 11392.0 | inactive_split_MB: 89.1 +rank3 [2026-10-27 21:33:08,656] INFO : Step 92000/100000 rank3 | Train Loss: 14.4974 | Train Loss Unweighted: 4.7629 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4460 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5917 | Regression_eta: 0.0739 | Regression_pt: 0.5467 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6297 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4340 | Classification_binary: 3.4301 | Regression_cos_phi: 1.5475 | Regression_energy: 2.8505 | Regression_eta: 0.6731 | Regression_pt: 4.0912 | Regression_sin_phi: 1.2940 | LR: 4.57e-05 | DataLoad Time: 0.0020s | Model Forward Time: 2.6195s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1620 | max_loss: 0.0037 | max_backward: 0.2478 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 1.1676 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0595 | Memory: rss_MB: 7302.2 | uss_MB: 6481.6 | child_rss_MB: 6997.3 | allocated_MB: 394.0 | reserved_MB: 11392.0 | inactive_split_MB: 90.6 +rank3 [2026-10-27 21:36:11,295] INFO : Step 93000/100000 rank3 | Train Loss: 13.1041 | Train Loss Unweighted: 4.5189 | Train Loss Components: Classification: 0.0119 | Classification_binary: 3.5265 | Regression_cos_phi: 0.0575 | Regression_energy: 0.4342 | Regression_eta: 0.0649 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6346 | physical_nopu: 4.4327 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2648 | Classification_binary: 3.4727 | Regression_cos_phi: 3.3894 | Regression_energy: 3.8375 | Regression_eta: 0.5871 | Regression_pt: 5.3202 | Regression_sin_phi: 2.4444 | LR: 4.43e-05 | DataLoad Time: 0.0021s | Model Forward Time: 2.4220s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1013 | max_loss: 0.1069 | max_backward: 0.2035 | max_optimizer_step: 0.1792 | max_tensorboard_logging: 1.1596 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 5379.1 | uss_MB: 5141.2 | child_rss_MB: 6951.6 | allocated_MB: 393.6 | reserved_MB: 13294.0 | inactive_split_MB: 89.2 +rank3 [2026-10-27 21:39:11,303] INFO : Step 94000/100000 rank3 | Train Loss: 11.5177 | Train Loss Unweighted: 4.2756 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4441 | Regression_cos_phi: 0.0413 | Regression_energy: 0.3503 | Regression_eta: 0.0686 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6308 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3150 | Classification_binary: 3.5417 | Regression_cos_phi: 1.4285 | Regression_energy: 1.6710 | Regression_eta: 1.0560 | Regression_pt: 2.2368 | Regression_sin_phi: 1.8403 | LR: 4.32e-05 | DataLoad Time: 0.0020s | Model Forward Time: 6.3166s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0191 | max_forward: 0.1721 | max_loss: 0.0188 | max_backward: 0.2028 | max_optimizer_step: 0.2895 | max_tensorboard_logging: 4.7123 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 5372.1 | uss_MB: 5157.2 | child_rss_MB: 6925.7 | allocated_MB: 394.1 | reserved_MB: 13294.0 | inactive_split_MB: 89.1 +rank3 [2026-10-27 21:42:20,694] INFO : Step 95000/100000 rank3 | Train Loss: 14.1207 | Train Loss Unweighted: 4.6477 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3920 | Regression_cos_phi: 0.0441 | Regression_energy: 0.5776 | Regression_eta: 0.0602 | Regression_pt: 0.5087 | Regression_sin_phi: 0.0499 | Group Train Loss Unweighted (window mean): physical_pu: 3.6267 | physical_nopu: 4.4321 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3951 | Classification_binary: 3.2724 | Regression_cos_phi: 1.2516 | Regression_energy: 6.3425 | Regression_eta: 0.8649 | Regression_pt: 7.4453 | Regression_sin_phi: 3.4572 | LR: 4.22e-05 | DataLoad Time: 0.0021s | Model Forward Time: 2.5032s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.0941 | max_loss: 0.0036 | max_backward: 0.1734 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 1.2811 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5361.0 | uss_MB: 5157.8 | child_rss_MB: 6928.3 | allocated_MB: 396.0 | reserved_MB: 14294.0 | inactive_split_MB: 91.1 +rank3 [2026-10-27 21:45:16,251] INFO : Step 96000/100000 rank3 | Train Loss: 11.5036 | Train Loss Unweighted: 4.2276 | Train Loss Components: Classification: 0.0097 | Classification_binary: 3.4897 | Regression_cos_phi: 0.0538 | Regression_energy: 0.2982 | Regression_eta: 0.0802 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6278 | physical_nopu: 4.4329 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1968 | Classification_binary: 3.3958 | Regression_cos_phi: 2.4936 | Regression_energy: 1.2888 | Regression_eta: 0.7348 | Regression_pt: 1.5054 | Regression_sin_phi: 1.5089 | LR: 4.14e-05 | DataLoad Time: 0.0023s | Model Forward Time: 2.3868s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0007 | max_forward: 0.0991 | max_loss: 0.0037 | max_backward: 0.1000 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5357.0 | uss_MB: 5158.4 | child_rss_MB: 6899.7 | allocated_MB: 393.0 | reserved_MB: 14294.0 | inactive_split_MB: 95.3 +rank3 [2026-10-27 21:48:34,573] INFO : Step 97000/100000 rank3 | Train Loss: 11.8793 | Train Loss Unweighted: 4.3044 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.4420 | Regression_cos_phi: 0.0454 | Regression_energy: 0.3663 | Regression_eta: 0.0711 | Regression_pt: 0.3156 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.6277 | physical_nopu: 4.4255 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4301 | Classification_binary: 3.2751 | Regression_cos_phi: 1.7561 | Regression_energy: 1.8291 | Regression_eta: 0.8408 | Regression_pt: 2.3095 | Regression_sin_phi: 1.6118 | LR: 4.08e-05 | DataLoad Time: 0.0009s | Model Forward Time: 2.5924s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1192 | max_loss: 0.0048 | max_backward: 0.1852 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 5356.0 | uss_MB: 5157.9 | child_rss_MB: 6903.2 | allocated_MB: 394.7 | reserved_MB: 14294.0 | inactive_split_MB: 98.1 +rank3 [2026-10-27 21:51:32,851] INFO : Step 98000/100000 rank3 | Train Loss: 12.9367 | Train Loss Unweighted: 4.4840 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0441 | Regression_energy: 0.4811 | Regression_eta: 0.0611 | Regression_pt: 0.4378 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3300 | Classification_binary: 3.3841 | Regression_cos_phi: 1.8576 | Regression_energy: 2.7075 | Regression_eta: 0.7138 | Regression_pt: 3.6713 | Regression_sin_phi: 1.7727 | LR: 4.04e-05 | DataLoad Time: 0.0009s | Model Forward Time: 2.8952s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1318 | max_loss: 0.0038 | max_backward: 0.1127 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.6868 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 5362.0 | uss_MB: 5160.3 | child_rss_MB: 6899.7 | allocated_MB: 394.5 | reserved_MB: 14294.0 | inactive_split_MB: 82.0 +rank3 [2026-10-27 21:54:35,739] INFO : Step 99000/100000 rank3 | Train Loss: 11.6408 | Train Loss Unweighted: 4.3204 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.5557 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3117 | Regression_eta: 0.0641 | Regression_pt: 0.2727 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.6292 | physical_nopu: 4.4284 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3087 | Classification_binary: 3.3450 | Regression_cos_phi: 1.7263 | Regression_energy: 1.6559 | Regression_eta: 0.7190 | Regression_pt: 2.2157 | Regression_sin_phi: 1.6565 | LR: 4.01e-05 | DataLoad Time: 0.0073s | Model Forward Time: 3.0816s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1731 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.8331 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 5358.0 | uss_MB: 5160.5 | child_rss_MB: 6886.0 | allocated_MB: 394.3 | reserved_MB: 14294.0 | inactive_split_MB: 113.7 +rank3 [2026-10-27 21:57:56,502] INFO : Step 100000/100000 rank3 | Train Loss: 10.8909 | Train Loss Unweighted: 3.6545 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9133 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2833 | Regression_eta: 0.1041 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6336 | physical_nopu: 4.4311 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3649 | Classification_binary: 2.8736 | Regression_cos_phi: 1.7206 | Regression_energy: 1.3238 | Regression_eta: 1.2770 | Regression_pt: 1.6066 | Regression_sin_phi: 1.6679 | LR: 4.00e-05 | DataLoad Time: 0.0021s | Model Forward Time: 3.7392s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0030 | max_forward: 0.2088 | max_loss: 0.0393 | max_backward: 0.3031 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.9276 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 5350.9 | uss_MB: 5163.3 | child_rss_MB: 6834.2 | allocated_MB: 405.7 | reserved_MB: 14294.0 | inactive_split_MB: 165.9 +rank3 [2026-10-27 21:57:57,456] INFO : Running validation on rank3 +rank3 [2026-10-27 21:57:57,456] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 21:57:57,456] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 21:57:57,456] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 21:57:57,456] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 21:59:51,633] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 21:59:52,049] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 22:00:18,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 22:00:18,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 22:00:19,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 22:00:19,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 22:00:20,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 22:00:20,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 22:00:21,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 22:00:21,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 22:00:31,874] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 22:00:32,061] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 22:00:32,465] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 22:00:47,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 22:00:48,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 22:00:48,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 22:00:49,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 22:00:49,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 22:00:50,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 22:00:50,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 22:00:51,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 22:01:01,203] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 22:01:01,522] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 22:01:01,928] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 22:01:16,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 22:01:17,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 22:01:17,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 22:01:18,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 22:01:18,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 22:01:19,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 22:01:19,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 22:01:20,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 22:01:30,106] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 22:01:31,011] INFO : Training completed. Total time on device 3: 350.239min diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.4 b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.4 new file mode 100644 index 0000000000000000000000000000000000000000..0bc09393faec9ade11a019196feb59a742a56eaf --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.4 @@ -0,0 +1,851 @@ +rank4 [2026-10-27 16:10:10,405] INFO : Initializing model with seed=12345; process seed=12349 +rank4 [2026-10-27 16:10:10,406] INFO : configured dtype=torch.bfloat16 for autocast +rank4 [2026-10-27 16:10:37,169] INFO : Instantiating model +rank4 [2026-10-27 16:10:37,169] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank4 [2026-10-27 16:10:37,169] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank4 [2026-10-27 16:10:37,201] INFO : Input encoding initialization took 0.03s +rank4 [2026-10-27 16:10:37,201] INFO : nn0 parameters: 2130944 +rank4 [2026-10-27 16:10:37,201] INFO : Initializing shared backbone layers, num_convs=3 +rank4 [2026-10-27 16:10:37,201] INFO : Initializing backbone layer 0 +rank4 [2026-10-27 16:10:37,201] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-27 16:10:37,212] INFO : Initializing backbone layer 1 +rank4 [2026-10-27 16:10:37,212] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-27 16:10:37,222] INFO : Initializing backbone layer 2 +rank4 [2026-10-27 16:10:37,222] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-27 16:10:37,232] INFO : Backbone initialization took 0.03s +rank4 [2026-10-27 16:10:37,233] INFO : backbone parameters: 4733952 +rank4 [2026-10-27 16:10:37,233] INFO : Initializing output DNNs +rank4 [2026-10-27 16:10:37,266] INFO : Output DNNs initialization took 0.03s +rank4 [2026-10-27 16:10:37,266] INFO : backbone_mode=BackboneMode.SHARED +rank4 [2026-10-27 16:10:37,266] INFO : jagged_attention enabled=True +rank4 [2026-10-27 16:10:37,267] INFO : flash_attn_varlen enabled=True +rank4 [2026-10-27 16:10:37,267] INFO : task_queries enabled=False +rank4 [2026-10-27 16:10:37,267] INFO : classification_norm parameters: 1024 +rank4 [2026-10-27 16:10:37,267] INFO : regression_norm parameters: 1024 +rank4 [2026-10-27 16:10:37,267] INFO : classification_query parameters: 0 +rank4 [2026-10-27 16:10:37,267] INFO : regression_query parameters: 0 +rank4 [2026-10-27 16:10:37,267] INFO : classification_readout parameters: 0 +rank4 [2026-10-27 16:10:37,267] INFO : regression_readout parameters: 0 +rank4 [2026-10-27 16:10:37,267] INFO : output_mode=elementwise +rank4 [2026-10-27 16:10:37,267] INFO : set_decoder parameters: 0 +rank4 [2026-10-27 16:10:37,267] INFO : nn_binary_particle parameters: 264706 +rank4 [2026-10-27 16:10:37,267] INFO : nn_pid parameters: 268297 +rank4 [2026-10-27 16:10:37,267] INFO : nn_pt parameters: 2113544 +rank4 [2026-10-27 16:10:37,267] INFO : nn_eta parameters: 264706 +rank4 [2026-10-27 16:10:37,267] INFO : nn_sin_phi parameters: 264706 +rank4 [2026-10-27 16:10:37,267] INFO : nn_cos_phi parameters: 264706 +rank4 [2026-10-27 16:10:37,268] INFO : nn_energy parameters: 2113544 +rank4 [2026-10-27 16:10:37,268] INFO : Total MLPF parameters: 12421153 +rank4 [2026-10-27 16:10:37,268] INFO : MLPF __init__ done +rank4 [2026-10-27 16:10:37,270] INFO : Instantiated model +rank4 [2026-10-27 16:10:37,270] INFO : Moving model to device rank=4 +rank4 [2026-10-27 16:10:37,292] INFO : Moved model to device rank=4 +rank4 [2026-10-27 16:10:37,294] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank4 [2026-10-27 16:10:37,294] INFO : Configured model for SyncBatchNorm rank=4 +rank4 [2026-10-27 16:10:37,488] INFO : Configured model for DistributedDataParallel rank=4 +rank4 [2026-10-27 16:10:37,593] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +167 module.nn_energy.nn.7.0.bias ... 0 +168 module.nn_energy.nn.7.2.weight ... 0 +169 module.nn_energy.nn.7.2.bias ... 0 +170 module.nn_energy.nn.7.4.weight ... 0 +171 module.nn_energy.nn.7.4.bias ... 0 + +[172 rows x 3 columns] +rank4 [2026-10-27 16:10:37,593] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=512, out_features=512, bias=True) + ) + (norm0): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=512, out_features=512, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank4 [2026-10-27 16:10:37,594] INFO : Trainable parameters: 12421153 +rank4 [2026-10-27 16:10:37,594] INFO : Non-trainable parameters: 0 +rank4 [2026-10-27 16:10:37,594] INFO : Total parameters: 12421153 +rank4 [2026-10-27 16:10:37,596] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.mha.in_proj_weight 786432 0 + module.backbone.0.mha.in_proj_bias 1536 0 +module.backbone.0.mha.out_proj.weight 262144 0 + module.backbone.0.mha.out_proj.bias 512 0 + module.backbone.0.norm0.weight 512 0 + module.backbone.0.norm0.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm1.bias 512 0 + module.backbone.0.seq.0.weight 262144 0 + module.backbone.0.seq.0.bias 512 0 + module.backbone.0.seq.2.weight 262144 0 + module.backbone.0.seq.2.bias 512 0 + module.backbone.1.mha.in_proj_weight 786432 0 + module.backbone.1.mha.in_proj_bias 1536 0 +module.backbone.1.mha.out_proj.weight 262144 0 + module.backbone.1.mha.out_proj.bias 512 0 + module.backbone.1.norm0.weight 512 0 + module.backbone.1.norm0.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm1.bias 512 0 + module.backbone.1.seq.0.weight 262144 0 + module.backbone.1.seq.0.bias 512 0 + module.backbone.1.seq.2.weight 262144 0 + module.backbone.1.seq.2.bias 512 0 + module.backbone.2.mha.in_proj_weight 786432 0 + module.backbone.2.mha.in_proj_bias 1536 0 +module.backbone.2.mha.out_proj.weight 262144 0 + module.backbone.2.mha.out_proj.bias 512 0 + module.backbone.2.norm0.weight 512 0 + module.backbone.2.norm0.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm1.bias 512 0 + module.backbone.2.seq.0.weight 262144 0 + module.backbone.2.seq.0.bias 512 0 + module.backbone.2.seq.2.weight 262144 0 + module.backbone.2.seq.2.bias 512 0 + module.classification_norm.weight 512 0 + module.classification_norm.bias 512 0 + module.regression_norm.weight 512 0 + module.regression_norm.bias 512 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank4 [2026-10-27 16:10:37,596] INFO : Getting dataloaders +rank4 [2026-10-27 16:10:37,596] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:43,313] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:44,237] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:44,827] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank4 [2026-10-27 16:10:44,827] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:46,010] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:46,995] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:47,802] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank4 [2026-10-27 16:10:47,802] INFO : Creating WeightedMixIterator with 2 data loaders. +rank4 [2026-10-27 16:10:47,802] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:47,962] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:48,111] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:48,259] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank4 [2026-10-27 16:10:48,259] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:48,422] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:48,584] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:48,746] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank4 [2026-10-27 16:10:48,747] INFO : Creating InterleavedIterator with 2 data loaders. +rank4 [2026-10-27 16:11:16,469] INFO : Got dataloaders +rank4 [2026-10-27 16:11:16,469] INFO : loader split=train rank=4 len=232058 +rank4 [2026-10-27 16:11:16,469] INFO : loader split=valid rank=4 len=48 +rank4 [2026-10-27 16:11:16,469] INFO : Starting training from step 1 to 100000 on rank 4 of 8 +rank4 [2026-10-27 16:11:16,469] INFO : Creating train_iterator +rank4 [2026-10-27 16:11:16,469] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank4 [2026-10-27 16:11:30,375] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank4 [2026-10-27 16:11:45,979] INFO : Created train_iterator +rank4 [2026-10-27 16:12:23,647] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.123548031449318, 'Classification': 0.2298217851272784, 'Regression_pt': 0.7194055180624127, 'Regression_eta': 0.4619348940253258, 'Regression_sin_phi': 0.15005176125559955, 'Regression_cos_phi': 0.14165307599119842, 'Regression_energy': 1.0569261094555258}, weights={'Classification_binary': 1.0, 'Classification': 22.293569564819336, 'Regression_pt': 7.12191915512085, 'Regression_eta': 11.091493606567383, 'Regression_sin_phi': 34.14520263671875, 'Regression_cos_phi': 36.16969299316406, 'Regression_energy': 4.847593307495117} +rank4 [2026-10-27 16:14:48,546] INFO : Step 1000/100000 rank4 | Train Loss: 15.9738 | Train Loss Unweighted: 5.5021 | Train Loss Components: Classification: 0.0191 | Classification_binary: 4.1436 | Regression_cos_phi: 0.0516 | Regression_energy: 0.6035 | Regression_eta: 0.0819 | Regression_pt: 0.5500 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 5.2846 | physical_nopu: 6.1663 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3717 | Classification_binary: 4.2073 | Regression_cos_phi: 1.5711 | Regression_energy: 2.7272 | Regression_eta: 1.0392 | Regression_pt: 3.6351 | Regression_sin_phi: 2.2645 | LR: 4.00e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.9584s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0013 | max_forward: 0.0194 | max_loss: 0.0039 | max_backward: 0.0308 | max_optimizer_step: 0.1391 | max_tensorboard_logging: 1.0816 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0030 | Memory: rss_MB: 4094.8 | uss_MB: 2885.6 | child_rss_MB: 10815.8 | allocated_MB: 395.5 | reserved_MB: 11198.0 | inactive_split_MB: 104.9 +rank4 [2026-10-27 16:17:31,218] INFO : Step 2000/100000 rank4 | Train Loss: 17.2752 | Train Loss Unweighted: 5.6831 | Train Loss Components: Classification: 0.0157 | Classification_binary: 4.0887 | Regression_cos_phi: 0.0506 | Regression_energy: 0.7236 | Regression_eta: 0.0696 | Regression_pt: 0.6798 | Regression_sin_phi: 0.0552 | Group Train Loss Unweighted (window mean): physical_pu: 4.4021 | physical_nopu: 5.4797 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2674 | Classification_binary: 4.0699 | Regression_cos_phi: 1.7984 | Regression_energy: 3.0802 | Regression_eta: 0.8097 | Regression_pt: 3.8198 | Regression_sin_phi: 1.6806 | LR: 4.00e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7790s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0250 | max_loss: 0.0036 | max_backward: 0.0657 | max_optimizer_step: 0.1397 | max_tensorboard_logging: 0.8949 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4097.7 | uss_MB: 2895.6 | child_rss_MB: 10834.7 | allocated_MB: 394.2 | reserved_MB: 11198.0 | inactive_split_MB: 90.1 +rank4 [2026-10-27 16:20:19,985] INFO : Step 3000/100000 rank4 | Train Loss: 14.0295 | Train Loss Unweighted: 5.1258 | Train Loss Components: Classification: 0.0148 | Classification_binary: 4.0405 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4749 | Regression_eta: 0.0746 | Regression_pt: 0.4205 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 4.1913 | physical_nopu: 5.3261 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3072 | Classification_binary: 4.1079 | Regression_cos_phi: 2.0026 | Regression_energy: 2.1720 | Regression_eta: 0.8287 | Regression_pt: 2.7580 | Regression_sin_phi: 1.8223 | LR: 3.99e-04 | DataLoad Time: 0.0026s | Model Forward Time: 2.4923s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1067 | max_loss: 0.0036 | max_backward: 0.0365 | max_optimizer_step: 0.1415 | max_tensorboard_logging: 1.1098 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0093 | Memory: rss_MB: 4102.6 | uss_MB: 2907.4 | child_rss_MB: 10837.4 | allocated_MB: 395.4 | reserved_MB: 11784.0 | inactive_split_MB: 88.5 +rank4 [2026-10-27 16:23:08,050] INFO : Step 4000/100000 rank4 | Train Loss: 11.7244 | Train Loss Unweighted: 3.9925 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.1718 | Regression_cos_phi: 0.0487 | Regression_energy: 0.3212 | Regression_eta: 0.1125 | Regression_pt: 0.2729 | Regression_sin_phi: 0.0492 | Group Train Loss Unweighted (window mean): physical_pu: 4.0838 | physical_nopu: 5.2370 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3753 | Classification_binary: 3.1868 | Regression_cos_phi: 1.6939 | Regression_energy: 1.4712 | Regression_eta: 1.2648 | Regression_pt: 1.8386 | Regression_sin_phi: 1.5891 | LR: 3.99e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.2691s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0037 | max_forward: 0.0726 | max_loss: 0.0066 | max_backward: 0.1670 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4108.6 | uss_MB: 2925.8 | child_rss_MB: 10838.2 | allocated_MB: 406.4 | reserved_MB: 11904.0 | inactive_split_MB: 179.6 +rank4 [2026-10-27 16:25:55,983] INFO : Step 5000/100000 rank4 | Train Loss: 13.1313 | Train Loss Unweighted: 4.9481 | Train Loss Components: Classification: 0.0138 | Classification_binary: 4.0144 | Regression_cos_phi: 0.0518 | Regression_energy: 0.3902 | Regression_eta: 0.0928 | Regression_pt: 0.3380 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 4.0057 | physical_nopu: 5.1582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2980 | Classification_binary: 4.0656 | Regression_cos_phi: 1.7390 | Regression_energy: 2.1366 | Regression_eta: 0.7888 | Regression_pt: 2.7533 | Regression_sin_phi: 1.5322 | LR: 3.98e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.3569s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0196 | max_loss: 0.0037 | max_backward: 0.0324 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 1.3896 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0068 | Memory: rss_MB: 4091.1 | uss_MB: 2927.1 | child_rss_MB: 10673.2 | allocated_MB: 393.9 | reserved_MB: 11904.0 | inactive_split_MB: 90.0 +rank4 [2026-10-27 16:28:58,362] INFO : Step 6000/100000 rank4 | Train Loss: 14.5226 | Train Loss Unweighted: 5.0639 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.8601 | Regression_cos_phi: 0.0525 | Regression_energy: 0.5359 | Regression_eta: 0.0694 | Regression_pt: 0.4825 | Regression_sin_phi: 0.0459 | Group Train Loss Unweighted (window mean): physical_pu: 3.9666 | physical_nopu: 5.0829 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4408 | Classification_binary: 3.7758 | Regression_cos_phi: 1.6529 | Regression_energy: 3.2798 | Regression_eta: 0.8382 | Regression_pt: 4.0868 | Regression_sin_phi: 1.3827 | LR: 3.97e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.5988s | Phase Times: max_pre_step_barrier: 0.0280 | max_batch_to_device: 0.0008 | max_forward: 0.0311 | max_loss: 0.0037 | max_backward: 0.0629 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 2.6604 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0119 | Memory: rss_MB: 4078.1 | uss_MB: 2934.4 | child_rss_MB: 10495.3 | allocated_MB: 394.6 | reserved_MB: 12524.0 | inactive_split_MB: 109.0 +rank4 [2026-10-27 16:31:51,869] INFO : Step 7000/100000 rank4 | Train Loss: 13.9772 | Train Loss Unweighted: 5.0398 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.9636 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4671 | Regression_eta: 0.0797 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0537 | Group Train Loss Unweighted (window mean): physical_pu: 3.9390 | physical_nopu: 5.0316 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3329 | Classification_binary: 3.9969 | Regression_cos_phi: 1.8216 | Regression_energy: 1.7986 | Regression_eta: 0.7357 | Regression_pt: 2.3131 | Regression_sin_phi: 1.5916 | LR: 3.96e-04 | DataLoad Time: 0.0010s | Model Forward Time: 3.1749s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0243 | max_loss: 0.0036 | max_backward: 0.0295 | max_optimizer_step: 0.2429 | max_tensorboard_logging: 2.1406 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 4070.6 | uss_MB: 2937.8 | child_rss_MB: 10370.1 | allocated_MB: 393.9 | reserved_MB: 12524.0 | inactive_split_MB: 107.5 +rank4 [2026-10-27 16:35:03,086] INFO : Step 8000/100000 rank4 | Train Loss: 13.6161 | Train Loss Unweighted: 4.9650 | Train Loss Components: Classification: 0.0187 | Classification_binary: 3.9275 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4519 | Regression_eta: 0.0615 | Regression_pt: 0.4056 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.9031 | physical_nopu: 4.9878 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4262 | Classification_binary: 4.0421 | Regression_cos_phi: 1.9535 | Regression_energy: 2.0989 | Regression_eta: 0.6267 | Regression_pt: 2.7782 | Regression_sin_phi: 1.6791 | LR: 3.94e-04 | DataLoad Time: 0.0048s | Model Forward Time: 2.9543s | Phase Times: max_pre_step_barrier: 0.0692 | max_batch_to_device: 0.0893 | max_forward: 0.1169 | max_loss: 0.0037 | max_backward: 0.0488 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 1.9012 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4069.6 | uss_MB: 2945.4 | child_rss_MB: 10376.1 | allocated_MB: 393.8 | reserved_MB: 12524.0 | inactive_split_MB: 105.1 +rank4 [2026-10-27 16:38:07,477] INFO : Step 9000/100000 rank4 | Train Loss: 15.1606 | Train Loss Unweighted: 5.0855 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.7493 | Regression_cos_phi: 0.0474 | Regression_energy: 0.5909 | Regression_eta: 0.0749 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.8927 | physical_nopu: 4.9579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3387 | Classification_binary: 3.6791 | Regression_cos_phi: 1.4932 | Regression_energy: 2.4138 | Regression_eta: 0.8541 | Regression_pt: 3.4000 | Regression_sin_phi: 1.5334 | LR: 3.93e-04 | DataLoad Time: 0.0058s | Model Forward Time: 2.8465s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0007 | max_forward: 0.0386 | max_loss: 0.0041 | max_backward: 0.0582 | max_optimizer_step: 0.1496 | max_tensorboard_logging: 1.9225 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 4056.6 | uss_MB: 2948.9 | child_rss_MB: 10378.8 | allocated_MB: 395.8 | reserved_MB: 12524.0 | inactive_split_MB: 89.5 +rank4 [2026-10-27 16:41:18,312] INFO : Step 10000/100000 rank4 | Train Loss: 12.2525 | Train Loss Unweighted: 4.6725 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8667 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3374 | Regression_eta: 0.0750 | Regression_pt: 0.2772 | Regression_sin_phi: 0.0525 | Group Train Loss Unweighted (window mean): physical_pu: 3.8762 | physical_nopu: 4.9262 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2530 | Classification_binary: 3.8015 | Regression_cos_phi: 2.1545 | Regression_energy: 1.3051 | Regression_eta: 0.5875 | Regression_pt: 1.6615 | Regression_sin_phi: 1.7888 | LR: 3.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.1907s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0277 | max_loss: 0.0035 | max_backward: 0.0304 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 2.2107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4024.3 | uss_MB: 2953.8 | child_rss_MB: 10382.4 | allocated_MB: 393.6 | reserved_MB: 12524.0 | inactive_split_MB: 89.8 +rank4 [2026-10-27 16:41:19,252] INFO : Running validation on rank4 +rank4 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 16:43:24,652] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:43:25,081] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 16:43:52,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 16:43:53,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 16:43:53,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 16:43:54,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 16:43:54,515] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 16:43:54,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 16:43:55,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 16:43:55,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 16:44:06,005] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-27 16:44:06,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:44:06,938] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 16:44:22,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 16:44:23,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 16:44:23,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 16:44:23,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 16:44:24,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 16:44:24,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 16:44:25,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 16:44:25,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 16:44:35,756] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 16:44:35,783] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:44:36,273] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 16:44:51,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 16:44:51,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 16:44:52,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 16:44:52,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 16:44:53,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 16:44:53,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 16:44:54,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 16:44:54,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 16:45:04,534] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 16:48:51,455] INFO : Step 11000/100000 rank4 | Train Loss: 13.5331 | Train Loss Unweighted: 4.8490 | Train Loss Components: Classification: 0.0178 | Classification_binary: 3.8332 | Regression_cos_phi: 0.0514 | Regression_energy: 0.4326 | Regression_eta: 0.0715 | Regression_pt: 0.3906 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.8629 | physical_nopu: 4.8860 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3906 | Classification_binary: 3.8973 | Regression_cos_phi: 1.9492 | Regression_energy: 1.8757 | Regression_eta: 0.6125 | Regression_pt: 2.5222 | Regression_sin_phi: 1.5565 | LR: 3.89e-04 | DataLoad Time: 0.0014s | Model Forward Time: 3.1314s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0010 | max_forward: 0.0938 | max_loss: 0.0036 | max_backward: 0.1077 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.9405 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6748.2 | uss_MB: 5693.6 | child_rss_MB: 10378.2 | allocated_MB: 394.1 | reserved_MB: 11016.0 | inactive_split_MB: 102.9 +rank4 [2026-10-27 16:51:46,972] INFO : Step 12000/100000 rank4 | Train Loss: 16.0880 | Train Loss Unweighted: 5.2955 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.7102 | Regression_cos_phi: 0.0409 | Regression_energy: 0.7300 | Regression_eta: 0.0621 | Regression_pt: 0.6963 | Regression_sin_phi: 0.0392 | Group Train Loss Unweighted (window mean): physical_pu: 3.8522 | physical_nopu: 4.8676 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4184 | Classification_binary: 3.8636 | Regression_cos_phi: 2.1424 | Regression_energy: 3.0078 | Regression_eta: 0.7927 | Regression_pt: 4.2672 | Regression_sin_phi: 1.6330 | LR: 3.87e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.9530s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1050 | max_forward: 0.2073 | max_loss: 0.0038 | max_backward: 0.1040 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 1.6163 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6734.2 | uss_MB: 5695.7 | child_rss_MB: 10317.1 | allocated_MB: 394.0 | reserved_MB: 11384.0 | inactive_split_MB: 88.0 +rank4 [2026-10-27 16:54:47,130] INFO : Step 13000/100000 rank4 | Train Loss: 12.2526 | Train Loss Unweighted: 4.6994 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.8775 | Regression_cos_phi: 0.0480 | Regression_energy: 0.3426 | Regression_eta: 0.0688 | Regression_pt: 0.2970 | Regression_sin_phi: 0.0539 | Group Train Loss Unweighted (window mean): physical_pu: 3.8404 | physical_nopu: 4.8415 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2445 | Classification_binary: 3.8217 | Regression_cos_phi: 1.3713 | Regression_energy: 1.7351 | Regression_eta: 0.7346 | Regression_pt: 2.2063 | Regression_sin_phi: 1.4998 | LR: 3.85e-04 | DataLoad Time: 0.0047s | Model Forward Time: 3.3101s | Phase Times: max_pre_step_barrier: 0.0143 | max_batch_to_device: 0.0010 | max_forward: 0.0740 | max_loss: 0.0036 | max_backward: 0.0639 | max_optimizer_step: 0.2642 | max_tensorboard_logging: 2.1016 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 5345.9 | uss_MB: 4318.0 | child_rss_MB: 10308.5 | allocated_MB: 394.7 | reserved_MB: 12168.0 | inactive_split_MB: 111.5 +rank4 [2026-10-27 16:57:46,633] INFO : Step 14000/100000 rank4 | Train Loss: 11.1328 | Train Loss Unweighted: 3.8032 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.0595 | Regression_cos_phi: 0.0494 | Regression_energy: 0.2823 | Regression_eta: 0.1077 | Regression_pt: 0.2396 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.8281 | physical_nopu: 4.8178 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3562 | Classification_binary: 3.1440 | Regression_cos_phi: 1.8568 | Regression_energy: 1.3461 | Regression_eta: 1.2012 | Regression_pt: 1.6592 | Regression_sin_phi: 1.6649 | LR: 3.83e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.7657s | Phase Times: max_pre_step_barrier: 0.1064 | max_batch_to_device: 0.0033 | max_forward: 0.1130 | max_loss: 0.0036 | max_backward: 0.2788 | max_optimizer_step: 0.1663 | max_tensorboard_logging: 2.1758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1099 | Memory: rss_MB: 5335.9 | uss_MB: 4329.8 | child_rss_MB: 10259.4 | allocated_MB: 405.3 | reserved_MB: 12168.0 | inactive_split_MB: 126.3 +rank4 [2026-10-27 17:00:50,805] INFO : Step 15000/100000 rank4 | Train Loss: 12.4223 | Train Loss Unweighted: 4.6301 | Train Loss Components: Classification: 0.0132 | Classification_binary: 3.7975 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3484 | Regression_eta: 0.0765 | Regression_pt: 0.2883 | Regression_sin_phi: 0.0502 | Group Train Loss Unweighted (window mean): physical_pu: 3.8205 | physical_nopu: 4.8025 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2712 | Classification_binary: 3.7780 | Regression_cos_phi: 2.3667 | Regression_energy: 2.2401 | Regression_eta: 1.0406 | Regression_pt: 2.4916 | Regression_sin_phi: 1.9503 | LR: 3.80e-04 | DataLoad Time: 0.0009s | Model Forward Time: 3.2237s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0799 | max_loss: 0.0851 | max_backward: 0.1383 | max_optimizer_step: 0.1621 | max_tensorboard_logging: 2.1079 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5320.9 | uss_MB: 4324.4 | child_rss_MB: 10137.7 | allocated_MB: 394.1 | reserved_MB: 12168.0 | inactive_split_MB: 89.6 +rank4 [2026-10-27 17:04:12,219] INFO : Step 16000/100000 rank4 | Train Loss: 13.1559 | Train Loss Unweighted: 4.6950 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.6917 | Regression_cos_phi: 0.0473 | Regression_energy: 0.4330 | Regression_eta: 0.0771 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.8147 | physical_nopu: 4.7795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2834 | Classification_binary: 3.6941 | Regression_cos_phi: 1.4783 | Regression_energy: 1.6360 | Regression_eta: 0.4850 | Regression_pt: 2.2214 | Regression_sin_phi: 1.6336 | LR: 3.78e-04 | DataLoad Time: 0.0031s | Model Forward Time: 3.2746s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.0598 | max_loss: 0.0036 | max_backward: 0.1881 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 2.1012 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5304.0 | uss_MB: 4326.3 | child_rss_MB: 10027.9 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 88.9 +rank4 [2026-10-27 17:07:14,836] INFO : Step 17000/100000 rank4 | Train Loss: 13.6189 | Train Loss Unweighted: 4.8180 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7348 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4756 | Regression_eta: 0.0731 | Regression_pt: 0.4225 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.8095 | physical_nopu: 4.7681 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4037 | Classification_binary: 3.7847 | Regression_cos_phi: 1.4568 | Regression_energy: 2.9009 | Regression_eta: 0.8417 | Regression_pt: 3.9057 | Regression_sin_phi: 1.5407 | LR: 3.75e-04 | DataLoad Time: 0.0021s | Model Forward Time: 4.0403s | Phase Times: max_pre_step_barrier: 0.6520 | max_batch_to_device: 0.0829 | max_forward: 0.1388 | max_loss: 0.0036 | max_backward: 0.0636 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 2.2240 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5273.0 | uss_MB: 4329.2 | child_rss_MB: 9776.7 | allocated_MB: 394.3 | reserved_MB: 12168.0 | inactive_split_MB: 88.4 +rank4 [2026-10-27 17:10:19,258] INFO : Step 18000/100000 rank4 | Train Loss: 12.6032 | Train Loss Unweighted: 4.6266 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7426 | Regression_cos_phi: 0.0507 | Regression_energy: 0.3730 | Regression_eta: 0.0736 | Regression_pt: 0.3195 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7954 | physical_nopu: 4.7518 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4234 | Classification_binary: 3.6127 | Regression_cos_phi: 1.6209 | Regression_energy: 1.7114 | Regression_eta: 0.8989 | Regression_pt: 2.2028 | Regression_sin_phi: 1.7536 | LR: 3.72e-04 | DataLoad Time: 0.0019s | Model Forward Time: 4.2069s | Phase Times: max_pre_step_barrier: 0.0217 | max_batch_to_device: 0.0009 | max_forward: 0.1212 | max_loss: 0.0040 | max_backward: 0.1670 | max_optimizer_step: 0.1610 | max_tensorboard_logging: 3.0399 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5245.9 | uss_MB: 4331.6 | child_rss_MB: 9623.0 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 87.1 +rank4 [2026-10-27 17:13:21,531] INFO : Step 19000/100000 rank4 | Train Loss: 14.6123 | Train Loss Unweighted: 5.0036 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.6735 | Regression_cos_phi: 0.0398 | Regression_energy: 0.6019 | Regression_eta: 0.0612 | Regression_pt: 0.5669 | Regression_sin_phi: 0.0441 | Group Train Loss Unweighted (window mean): physical_pu: 3.7887 | physical_nopu: 4.7426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3306 | Classification_binary: 3.6200 | Regression_cos_phi: 1.1222 | Regression_energy: 1.9728 | Regression_eta: 0.5466 | Regression_pt: 2.7183 | Regression_sin_phi: 1.6744 | LR: 3.69e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.4962s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1593 | max_loss: 0.0094 | max_backward: 0.0749 | max_optimizer_step: 0.1658 | max_tensorboard_logging: 2.3024 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5218.9 | uss_MB: 4333.7 | child_rss_MB: 9466.4 | allocated_MB: 394.5 | reserved_MB: 12168.0 | inactive_split_MB: 103.1 +rank4 [2026-10-27 17:16:34,024] INFO : Step 20000/100000 rank4 | Train Loss: 13.1046 | Train Loss Unweighted: 4.7571 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.7799 | Regression_cos_phi: 0.0517 | Regression_energy: 0.4251 | Regression_eta: 0.0654 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0511 | Group Train Loss Unweighted (window mean): physical_pu: 3.7835 | physical_nopu: 4.7206 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2624 | Classification_binary: 3.6445 | Regression_cos_phi: 2.0571 | Regression_energy: 3.6812 | Regression_eta: 0.7039 | Regression_pt: 5.0951 | Regression_sin_phi: 1.7451 | LR: 3.66e-04 | DataLoad Time: 0.0026s | Model Forward Time: 3.2228s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0008 | max_forward: 0.1344 | max_loss: 0.0037 | max_backward: 0.0763 | max_optimizer_step: 0.1649 | max_tensorboard_logging: 1.9693 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5216.3 | uss_MB: 4336.2 | child_rss_MB: 9462.6 | allocated_MB: 392.8 | reserved_MB: 12168.0 | inactive_split_MB: 89.4 +rank4 [2026-10-27 17:16:34,968] INFO : Running validation on rank4 +rank4 [2026-10-27 17:16:34,968] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 17:18:27,486] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 17:18:27,879] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 17:18:53,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 17:18:54,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 17:18:54,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 17:18:55,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 17:18:55,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 17:18:56,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 17:18:56,579] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 17:18:57,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 17:19:07,038] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 17:19:07,312] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 17:19:07,733] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 17:19:22,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 17:19:23,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 17:19:23,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 17:19:24,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 17:19:24,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 17:19:25,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 17:19:25,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 17:19:26,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 17:19:36,228] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 17:19:36,428] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 17:19:36,812] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 17:19:51,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 17:19:52,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 17:19:52,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 17:19:53,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 17:19:53,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 17:19:54,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 17:19:54,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 17:19:55,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 17:20:05,151] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 17:23:25,419] INFO : Step 21000/100000 rank4 | Train Loss: 12.4320 | Train Loss Unweighted: 4.5235 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.6041 | Regression_cos_phi: 0.0444 | Regression_energy: 0.3902 | Regression_eta: 0.0812 | Regression_pt: 0.3383 | Regression_sin_phi: 0.0478 | Group Train Loss Unweighted (window mean): physical_pu: 3.7732 | physical_nopu: 4.7052 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4042 | Classification_binary: 3.5764 | Regression_cos_phi: 1.8782 | Regression_energy: 1.8870 | Regression_eta: 0.7926 | Regression_pt: 2.4286 | Regression_sin_phi: 1.6315 | LR: 3.62e-04 | DataLoad Time: 0.0027s | Model Forward Time: 3.7286s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.2105 | max_loss: 0.0056 | max_backward: 0.1347 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 2.3606 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5326.2 | uss_MB: 4454.3 | child_rss_MB: 9450.4 | allocated_MB: 394.6 | reserved_MB: 11062.0 | inactive_split_MB: 89.8 +rank4 [2026-10-27 17:26:14,701] INFO : Step 22000/100000 rank4 | Train Loss: 14.5232 | Train Loss Unweighted: 4.8893 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.6034 | Regression_cos_phi: 0.0476 | Regression_energy: 0.5714 | Regression_eta: 0.0738 | Regression_pt: 0.5351 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.7709 | physical_nopu: 4.6938 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3295 | Classification_binary: 3.5790 | Regression_cos_phi: 1.5676 | Regression_energy: 1.8437 | Regression_eta: 0.6188 | Regression_pt: 2.3707 | Regression_sin_phi: 1.8594 | LR: 3.59e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.6771s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0010 | max_forward: 0.0774 | max_loss: 0.0142 | max_backward: 0.0921 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.4442 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1027 | Memory: rss_MB: 5327.2 | uss_MB: 4458.5 | child_rss_MB: 9447.9 | allocated_MB: 394.2 | reserved_MB: 11488.0 | inactive_split_MB: 86.8 +rank4 [2026-10-27 17:29:50,014] INFO : Step 23000/100000 rank4 | Train Loss: 12.2140 | Train Loss Unweighted: 4.5907 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.7888 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3388 | Regression_eta: 0.0762 | Regression_pt: 0.2687 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.7690 | physical_nopu: 4.6810 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2588 | Classification_binary: 3.8636 | Regression_cos_phi: 1.7752 | Regression_energy: 1.5882 | Regression_eta: 0.5987 | Regression_pt: 2.0220 | Regression_sin_phi: 1.5687 | LR: 3.55e-04 | DataLoad Time: 0.0106s | Model Forward Time: 4.1181s | Phase Times: max_pre_step_barrier: 0.7590 | max_batch_to_device: 0.0007 | max_forward: 0.0763 | max_loss: 0.0189 | max_backward: 0.1117 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 2.0913 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5325.2 | uss_MB: 4459.8 | child_rss_MB: 9460.6 | allocated_MB: 394.0 | reserved_MB: 12220.0 | inactive_split_MB: 178.1 +rank4 [2026-10-27 17:33:22,189] INFO : Step 24000/100000 rank4 | Train Loss: 15.4786 | Train Loss Unweighted: 4.9908 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.5301 | Regression_cos_phi: 0.0467 | Regression_energy: 0.6682 | Regression_eta: 0.0624 | Regression_pt: 0.6223 | Regression_sin_phi: 0.0450 | Group Train Loss Unweighted (window mean): physical_pu: 3.7564 | physical_nopu: 4.6761 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3630 | Classification_binary: 3.3861 | Regression_cos_phi: 1.6149 | Regression_energy: 2.9476 | Regression_eta: 0.5719 | Regression_pt: 4.1382 | Regression_sin_phi: 1.6970 | LR: 3.51e-04 | DataLoad Time: 0.0015s | Model Forward Time: 2.9320s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0719 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1672 | max_tensorboard_logging: 1.8279 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5318.8 | uss_MB: 4462.5 | child_rss_MB: 9451.5 | allocated_MB: 393.8 | reserved_MB: 12220.0 | inactive_split_MB: 105.1 +rank4 [2026-10-27 17:36:26,065] INFO : Step 25000/100000 rank4 | Train Loss: 12.1246 | Train Loss Unweighted: 4.4783 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.7007 | Regression_cos_phi: 0.0551 | Regression_energy: 0.3196 | Regression_eta: 0.0694 | Regression_pt: 0.2642 | Regression_sin_phi: 0.0578 | Group Train Loss Unweighted (window mean): physical_pu: 3.7531 | physical_nopu: 4.6582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2768 | Classification_binary: 3.6813 | Regression_cos_phi: 1.9994 | Regression_energy: 1.8846 | Regression_eta: 0.9440 | Regression_pt: 2.0970 | Regression_sin_phi: 1.9275 | LR: 3.47e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.0301s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.1917 | max_loss: 0.0039 | max_backward: 0.1029 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 5304.8 | uss_MB: 4464.1 | child_rss_MB: 9445.8 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 86.4 +rank4 [2026-10-27 17:39:30,100] INFO : Step 26000/100000 rank4 | Train Loss: 12.6920 | Train Loss Unweighted: 4.5479 | Train Loss Components: Classification: 0.0188 | Classification_binary: 3.6003 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4034 | Regression_eta: 0.0734 | Regression_pt: 0.3564 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.7511 | physical_nopu: 4.6505 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4806 | Classification_binary: 3.5024 | Regression_cos_phi: 1.3288 | Regression_energy: 1.7232 | Regression_eta: 0.7029 | Regression_pt: 2.2672 | Regression_sin_phi: 1.1721 | LR: 3.43e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.0940s | Phase Times: max_pre_step_barrier: 0.0479 | max_batch_to_device: 0.0008 | max_forward: 0.1278 | max_loss: 0.0037 | max_backward: 0.0877 | max_optimizer_step: 0.1868 | max_tensorboard_logging: 1.8908 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5292.2 | uss_MB: 4467.2 | child_rss_MB: 9407.7 | allocated_MB: 395.0 | reserved_MB: 12220.0 | inactive_split_MB: 115.0 +rank4 [2026-10-27 17:42:32,922] INFO : Step 27000/100000 rank4 | Train Loss: 16.2758 | Train Loss Unweighted: 5.1421 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5608 | Regression_cos_phi: 0.0448 | Regression_energy: 0.7180 | Regression_eta: 0.0780 | Regression_pt: 0.6791 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.7443 | physical_nopu: 4.6426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4373 | Classification_binary: 3.4127 | Regression_cos_phi: 1.7468 | Regression_energy: 4.6514 | Regression_eta: 0.9007 | Regression_pt: 6.5336 | Regression_sin_phi: 1.4948 | LR: 3.39e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.0080s | Phase Times: max_pre_step_barrier: 0.0385 | max_batch_to_device: 0.0009 | max_forward: 0.0708 | max_loss: 0.0038 | max_backward: 0.0914 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.8878 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5283.2 | uss_MB: 4469.8 | child_rss_MB: 9400.6 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 87.9 +rank4 [2026-10-27 17:45:38,004] INFO : Step 28000/100000 rank4 | Train Loss: 11.9115 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6175 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3339 | Regression_eta: 0.0761 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.7393 | physical_nopu: 4.6288 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2880 | Classification_binary: 3.4137 | Regression_cos_phi: 1.5730 | Regression_energy: 1.3616 | Regression_eta: 0.7193 | Regression_pt: 1.8527 | Regression_sin_phi: 1.3921 | LR: 3.35e-04 | DataLoad Time: 0.0018s | Model Forward Time: 4.8023s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0008 | max_forward: 0.0886 | max_loss: 0.0038 | max_backward: 0.1193 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 3.6285 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 5274.2 | uss_MB: 4146.2 | child_rss_MB: 9355.1 | allocated_MB: 394.0 | reserved_MB: 12414.0 | inactive_split_MB: 106.5 +rank4 [2026-10-27 17:48:54,715] INFO : Step 29000/100000 rank4 | Train Loss: 13.9221 | Train Loss Unweighted: 4.6615 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.5075 | Regression_cos_phi: 0.0491 | Regression_energy: 0.5008 | Regression_eta: 0.0786 | Regression_pt: 0.4589 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7405 | physical_nopu: 4.6233 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3204 | Classification_binary: 3.6359 | Regression_cos_phi: 1.5895 | Regression_energy: 2.3594 | Regression_eta: 0.7421 | Regression_pt: 3.0836 | Regression_sin_phi: 1.8893 | LR: 3.30e-04 | DataLoad Time: 0.0188s | Model Forward Time: 3.3644s | Phase Times: max_pre_step_barrier: 0.1677 | max_batch_to_device: 0.0009 | max_forward: 0.0754 | max_loss: 0.0038 | max_backward: 0.0825 | max_optimizer_step: 0.1673 | max_tensorboard_logging: 2.1272 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5252.2 | uss_MB: 4453.2 | child_rss_MB: 9260.7 | allocated_MB: 394.9 | reserved_MB: 12414.0 | inactive_split_MB: 107.1 +rank4 [2026-10-27 17:51:57,834] INFO : Step 30000/100000 rank4 | Train Loss: 12.4988 | Train Loss Unweighted: 4.5639 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.6431 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3981 | Regression_eta: 0.0722 | Regression_pt: 0.3405 | Regression_sin_phi: 0.0498 | Group Train Loss Unweighted (window mean): physical_pu: 3.7342 | physical_nopu: 4.6125 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3922 | Classification_binary: 3.7326 | Regression_cos_phi: 1.6820 | Regression_energy: 1.9131 | Regression_eta: 0.7841 | Regression_pt: 2.5632 | Regression_sin_phi: 1.8218 | LR: 3.26e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.4591s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0762 | max_loss: 0.0136 | max_backward: 0.0854 | max_optimizer_step: 0.1722 | max_tensorboard_logging: 1.3349 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5242.2 | uss_MB: 4475.0 | child_rss_MB: 9193.8 | allocated_MB: 393.7 | reserved_MB: 12480.0 | inactive_split_MB: 89.7 +rank4 [2026-10-27 17:51:58,780] INFO : Running validation on rank4 +rank4 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 17:51:58,781] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 17:51:58,781] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 17:53:53,311] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 17:53:53,722] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 17:54:18,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 17:54:18,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 17:54:19,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 17:54:19,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 17:54:20,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 17:54:20,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 17:54:21,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 17:54:21,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 17:54:31,734] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 17:54:31,840] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 17:54:32,213] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 17:54:47,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 17:54:47,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 17:54:48,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 17:54:48,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 17:54:49,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 17:54:49,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 17:54:49,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 17:54:50,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 17:55:00,422] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 17:55:00,637] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 17:55:01,041] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 17:55:16,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 17:55:17,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 17:55:17,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 17:55:18,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 17:55:18,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 17:55:19,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 17:55:19,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 17:55:20,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 17:55:30,167] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 17:58:47,285] INFO : Step 31000/100000 rank4 | Train Loss: 12.0027 | Train Loss Unweighted: 4.4503 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.5779 | Regression_cos_phi: 0.0460 | Regression_energy: 0.3731 | Regression_eta: 0.0704 | Regression_pt: 0.3230 | Regression_sin_phi: 0.0451 | Group Train Loss Unweighted (window mean): physical_pu: 3.7262 | physical_nopu: 4.6083 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3343 | Classification_binary: 3.6803 | Regression_cos_phi: 1.3436 | Regression_energy: 1.8251 | Regression_eta: 0.6633 | Regression_pt: 2.4801 | Regression_sin_phi: 1.1779 | LR: 3.21e-04 | DataLoad Time: 0.0008s | Model Forward Time: 3.5869s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.1458 | max_loss: 0.0037 | max_backward: 0.1256 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 2.3030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7018.1 | uss_MB: 6249.8 | child_rss_MB: 9197.9 | allocated_MB: 394.8 | reserved_MB: 11168.0 | inactive_split_MB: 96.2 +rank4 [2026-10-27 18:01:54,932] INFO : Step 32000/100000 rank4 | Train Loss: 13.5217 | Train Loss Unweighted: 4.6918 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.5357 | Regression_cos_phi: 0.0443 | Regression_energy: 0.5169 | Regression_eta: 0.0728 | Regression_pt: 0.4649 | Regression_sin_phi: 0.0409 | Group Train Loss Unweighted (window mean): physical_pu: 3.7264 | physical_nopu: 4.6006 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4195 | Classification_binary: 3.4482 | Regression_cos_phi: 1.8162 | Regression_energy: 3.4636 | Regression_eta: 0.6663 | Regression_pt: 4.6993 | Regression_sin_phi: 1.5025 | LR: 3.16e-04 | DataLoad Time: 0.0063s | Model Forward Time: 2.3920s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.0795 | max_loss: 0.0917 | max_backward: 0.1851 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.1890 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5310.0 | uss_MB: 4562.1 | child_rss_MB: 9157.4 | allocated_MB: 394.9 | reserved_MB: 11240.0 | inactive_split_MB: 92.3 +rank4 [2026-10-27 18:04:57,677] INFO : Step 33000/100000 rank4 | Train Loss: 11.0126 | Train Loss Unweighted: 3.7465 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9928 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2905 | Regression_eta: 0.1058 | Regression_pt: 0.2472 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.7276 | physical_nopu: 4.5909 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3518 | Classification_binary: 2.9756 | Regression_cos_phi: 1.6996 | Regression_energy: 1.3833 | Regression_eta: 1.1736 | Regression_pt: 1.6806 | Regression_sin_phi: 1.5607 | LR: 3.12e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.8905s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0027 | max_forward: 0.1127 | max_loss: 0.0150 | max_backward: 0.3396 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.4660 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5295.8 | uss_MB: 4291.2 | child_rss_MB: 9057.9 | allocated_MB: 404.8 | reserved_MB: 11922.0 | inactive_split_MB: 135.4 +rank4 [2026-10-27 18:07:50,121] INFO : Step 34000/100000 rank4 | Train Loss: 14.2400 | Train Loss Unweighted: 4.7634 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5051 | Regression_cos_phi: 0.0450 | Regression_energy: 0.5627 | Regression_eta: 0.0631 | Regression_pt: 0.5251 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7180 | physical_nopu: 4.5858 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4184 | Classification_binary: 3.4149 | Regression_cos_phi: 1.6687 | Regression_energy: 2.7438 | Regression_eta: 0.7849 | Regression_pt: 3.7791 | Regression_sin_phi: 1.9778 | LR: 3.07e-04 | DataLoad Time: 0.1789s | Model Forward Time: 2.5637s | Phase Times: max_pre_step_barrier: 0.1802 | max_batch_to_device: 0.0765 | max_forward: 0.1650 | max_loss: 0.0036 | max_backward: 0.0935 | max_optimizer_step: 0.2788 | max_tensorboard_logging: 1.2621 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5285.5 | uss_MB: 4563.5 | child_rss_MB: 9059.3 | allocated_MB: 395.5 | reserved_MB: 11922.0 | inactive_split_MB: 99.3 +rank4 [2026-10-27 18:10:53,637] INFO : Step 35000/100000 rank4 | Train Loss: 11.4005 | Train Loss Unweighted: 4.4142 | Train Loss Components: Classification: 0.0091 | Classification_binary: 3.6972 | Regression_cos_phi: 0.0530 | Regression_energy: 0.2984 | Regression_eta: 0.0671 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.7147 | physical_nopu: 4.5802 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2883 | Classification_binary: 3.7234 | Regression_cos_phi: 2.0860 | Regression_energy: 1.7017 | Regression_eta: 0.6847 | Regression_pt: 2.0844 | Regression_sin_phi: 1.6534 | LR: 3.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.4158s | Phase Times: max_pre_step_barrier: 0.0682 | max_batch_to_device: 0.0008 | max_forward: 0.1243 | max_loss: 0.0038 | max_backward: 0.0883 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.1949 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5274.5 | uss_MB: 4566.4 | child_rss_MB: 9063.3 | allocated_MB: 393.8 | reserved_MB: 11922.0 | inactive_split_MB: 85.2 +rank4 [2026-10-27 18:13:50,577] INFO : Step 36000/100000 rank4 | Train Loss: 11.8438 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.5751 | Regression_cos_phi: 0.0431 | Regression_energy: 0.3555 | Regression_eta: 0.0641 | Regression_pt: 0.3210 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.7110 | physical_nopu: 4.5729 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3988 | Classification_binary: 3.4185 | Regression_cos_phi: 1.4598 | Regression_energy: 1.6288 | Regression_eta: 0.6198 | Regression_pt: 2.1700 | Regression_sin_phi: 1.1754 | LR: 2.97e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.4512s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0993 | max_loss: 0.0066 | max_backward: 0.1760 | max_optimizer_step: 0.1735 | max_tensorboard_logging: 1.2166 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 5259.5 | uss_MB: 4568.3 | child_rss_MB: 9032.9 | allocated_MB: 395.0 | reserved_MB: 11922.0 | inactive_split_MB: 95.2 +rank4 [2026-10-27 18:16:45,596] INFO : Step 37000/100000 rank4 | Train Loss: 15.1390 | Train Loss Unweighted: 4.8166 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4396 | Regression_cos_phi: 0.0489 | Regression_energy: 0.6142 | Regression_eta: 0.0886 | Regression_pt: 0.5621 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7102 | physical_nopu: 4.5673 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4009 | Classification_binary: 3.2411 | Regression_cos_phi: 1.9081 | Regression_energy: 4.8325 | Regression_eta: 2.0558 | Regression_pt: 6.6644 | Regression_sin_phi: 2.3421 | LR: 2.91e-04 | DataLoad Time: 0.0011s | Model Forward Time: 2.2794s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1889 | max_loss: 0.0036 | max_backward: 0.0899 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 1.0901 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 5243.2 | uss_MB: 4569.9 | child_rss_MB: 9022.4 | allocated_MB: 394.3 | reserved_MB: 11922.0 | inactive_split_MB: 107.1 +rank4 [2026-10-27 18:19:42,075] INFO : Step 38000/100000 rank4 | Train Loss: 11.6256 | Train Loss Unweighted: 4.4424 | Train Loss Components: Classification: 0.0117 | Classification_binary: 3.6501 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3369 | Regression_eta: 0.0664 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7131 | physical_nopu: 4.5682 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2412 | Classification_binary: 3.8664 | Regression_cos_phi: 1.6556 | Regression_energy: 1.4347 | Regression_eta: 0.5933 | Regression_pt: 1.7353 | Regression_sin_phi: 1.8594 | LR: 2.86e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.2501s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.0965 | max_loss: 0.0038 | max_backward: 0.1051 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5237.2 | uss_MB: 4571.0 | child_rss_MB: 9030.3 | allocated_MB: 393.3 | reserved_MB: 12082.0 | inactive_split_MB: 110.5 +rank4 [2026-10-27 18:22:37,812] INFO : Step 39000/100000 rank4 | Train Loss: 13.0208 | Train Loss Unweighted: 4.4928 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4758 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4360 | Regression_eta: 0.0793 | Regression_pt: 0.3879 | Regression_sin_phi: 0.0500 | Group Train Loss Unweighted (window mean): physical_pu: 3.7060 | physical_nopu: 4.5607 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3544 | Classification_binary: 3.4837 | Regression_cos_phi: 1.5913 | Regression_energy: 1.9468 | Regression_eta: 1.1897 | Regression_pt: 2.6982 | Regression_sin_phi: 1.4279 | LR: 2.81e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.2313s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.0864 | max_loss: 0.0038 | max_backward: 0.0877 | max_optimizer_step: 0.1700 | max_tensorboard_logging: 1.1418 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5224.7 | uss_MB: 4572.8 | child_rss_MB: 9008.2 | allocated_MB: 394.4 | reserved_MB: 12706.0 | inactive_split_MB: 84.9 +rank4 [2026-10-27 18:25:31,277] INFO : Step 40000/100000 rank4 | Train Loss: 14.7562 | Train Loss Unweighted: 4.9007 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5708 | Regression_cos_phi: 0.0459 | Regression_energy: 0.5954 | Regression_eta: 0.0747 | Regression_pt: 0.5553 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7006 | physical_nopu: 4.5579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2497 | Classification_binary: 3.4609 | Regression_cos_phi: 1.5449 | Regression_energy: 2.3532 | Regression_eta: 0.9326 | Regression_pt: 3.1067 | Regression_sin_phi: 1.3838 | LR: 2.76e-04 | DataLoad Time: 0.0011s | Model Forward Time: 2.3081s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.0814 | max_loss: 0.0614 | max_backward: 0.1417 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 1.1378 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5222.7 | uss_MB: 4573.6 | child_rss_MB: 8992.8 | allocated_MB: 393.7 | reserved_MB: 12706.0 | inactive_split_MB: 107.7 +rank4 [2026-10-27 18:25:32,321] INFO : Running validation on rank4 +rank4 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 18:27:23,860] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 18:27:24,220] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 18:27:48,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 18:27:48,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 18:27:49,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 18:27:49,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 18:27:50,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 18:27:50,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 18:27:51,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 18:27:51,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 18:28:01,639] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 18:28:01,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 18:28:02,235] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 18:28:17,361] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 18:28:17,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 18:28:18,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 18:28:18,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 18:28:19,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 18:28:19,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 18:28:20,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 18:28:20,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 18:28:30,726] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 18:28:30,989] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 18:28:31,367] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 18:28:46,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 18:28:47,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 18:28:47,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 18:28:48,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 18:28:48,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 18:28:48,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 18:28:49,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 18:28:49,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 18:28:59,829] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 18:32:27,442] INFO : Step 41000/100000 rank4 | Train Loss: 12.3609 | Train Loss Unweighted: 4.4817 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5838 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3837 | Regression_eta: 0.0731 | Regression_pt: 0.3287 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.7016 | physical_nopu: 4.5433 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3145 | Classification_binary: 3.6423 | Regression_cos_phi: 1.6746 | Regression_energy: 1.8408 | Regression_eta: 0.5459 | Regression_pt: 2.4500 | Regression_sin_phi: 1.3191 | LR: 2.70e-04 | DataLoad Time: 0.0019s | Model Forward Time: 4.5172s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0009 | max_forward: 0.1338 | max_loss: 0.0038 | max_backward: 0.1857 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 3.1120 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 7028.8 | uss_MB: 6392.1 | child_rss_MB: 8998.3 | allocated_MB: 394.0 | reserved_MB: 11364.0 | inactive_split_MB: 106.1 +rank4 [2026-10-27 18:35:19,857] INFO : Step 42000/100000 rank4 | Train Loss: 12.9790 | Train Loss Unweighted: 4.5354 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4379 | Regression_cos_phi: 0.0458 | Regression_energy: 0.4856 | Regression_eta: 0.0625 | Regression_pt: 0.4500 | Regression_sin_phi: 0.0370 | Group Train Loss Unweighted (window mean): physical_pu: 3.6959 | physical_nopu: 4.5474 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3224 | Classification_binary: 3.5133 | Regression_cos_phi: 1.7376 | Regression_energy: 1.7475 | Regression_eta: 0.6067 | Regression_pt: 2.2481 | Regression_sin_phi: 1.1328 | LR: 2.65e-04 | DataLoad Time: 0.0105s | Model Forward Time: 2.6334s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0009 | max_forward: 0.1375 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 1.3492 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 7016.8 | uss_MB: 5332.4 | child_rss_MB: 8994.4 | allocated_MB: 394.2 | reserved_MB: 12150.0 | inactive_split_MB: 99.3 +rank4 [2026-10-27 18:38:35,926] INFO : Step 43000/100000 rank4 | Train Loss: 10.9006 | Train Loss Unweighted: 3.7057 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.9740 | Regression_cos_phi: 0.0473 | Regression_energy: 0.2786 | Regression_eta: 0.1094 | Regression_pt: 0.2322 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6942 | physical_nopu: 4.5421 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3257 | Classification_binary: 2.9661 | Regression_cos_phi: 1.6614 | Regression_energy: 1.2837 | Regression_eta: 1.1701 | Regression_pt: 1.5560 | Regression_sin_phi: 1.5780 | LR: 2.59e-04 | DataLoad Time: 0.0019s | Model Forward Time: 5.2293s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0039 | max_forward: 0.1100 | max_loss: 0.0637 | max_backward: 0.2303 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 3.9250 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5285.0 | uss_MB: 4686.1 | child_rss_MB: 8875.5 | allocated_MB: 406.8 | reserved_MB: 12150.0 | inactive_split_MB: 152.7 +rank4 [2026-10-27 18:41:39,775] INFO : Step 44000/100000 rank4 | Train Loss: 14.8629 | Train Loss Unweighted: 4.8399 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4856 | Regression_cos_phi: 0.0438 | Regression_energy: 0.6070 | Regression_eta: 0.0739 | Regression_pt: 0.5644 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6933 | physical_nopu: 4.5331 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3521 | Classification_binary: 3.4154 | Regression_cos_phi: 1.3864 | Regression_energy: 2.1104 | Regression_eta: 0.8335 | Regression_pt: 2.6682 | Regression_sin_phi: 1.3251 | LR: 2.54e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.6836s | Phase Times: max_pre_step_barrier: 0.0851 | max_batch_to_device: 0.0008 | max_forward: 0.0946 | max_loss: 0.0041 | max_backward: 0.0964 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.3558 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5284.0 | uss_MB: 4679.1 | child_rss_MB: 8885.5 | allocated_MB: 394.8 | reserved_MB: 12150.0 | inactive_split_MB: 94.9 +rank4 [2026-10-27 18:44:46,984] INFO : Step 45000/100000 rank4 | Train Loss: 12.9535 | Train Loss Unweighted: 4.5658 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.6115 | Regression_cos_phi: 0.0537 | Regression_energy: 0.4117 | Regression_eta: 0.0846 | Regression_pt: 0.3413 | Regression_sin_phi: 0.0531 | Group Train Loss Unweighted (window mean): physical_pu: 3.6892 | physical_nopu: 4.5283 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1896 | Classification_binary: 3.7330 | Regression_cos_phi: 2.0101 | Regression_energy: 1.4092 | Regression_eta: 0.7422 | Regression_pt: 1.9622 | Regression_sin_phi: 1.5092 | LR: 2.48e-04 | DataLoad Time: 0.0029s | Model Forward Time: 2.4260s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1077 | max_loss: 0.0040 | max_backward: 0.1457 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 1.2404 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 5275.4 | uss_MB: 4680.1 | child_rss_MB: 8893.2 | allocated_MB: 394.0 | reserved_MB: 13366.0 | inactive_split_MB: 74.2 +rank4 [2026-10-27 18:48:06,241] INFO : Step 46000/100000 rank4 | Train Loss: 12.3783 | Train Loss Unweighted: 4.4476 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.5252 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3923 | Regression_eta: 0.0679 | Regression_pt: 0.3506 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6869 | physical_nopu: 4.5281 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3049 | Classification_binary: 3.4301 | Regression_cos_phi: 1.6372 | Regression_energy: 1.9984 | Regression_eta: 0.8042 | Regression_pt: 2.5280 | Regression_sin_phi: 1.7612 | LR: 2.43e-04 | DataLoad Time: 0.0008s | Model Forward Time: 2.4384s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1021 | max_loss: 0.0036 | max_backward: 0.1895 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 1.1894 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 5269.4 | uss_MB: 4681.7 | child_rss_MB: 8897.5 | allocated_MB: 395.1 | reserved_MB: 13366.0 | inactive_split_MB: 103.1 +rank4 [2026-10-27 18:51:16,142] INFO : Step 47000/100000 rank4 | Train Loss: 14.5142 | Train Loss Unweighted: 4.7631 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.3922 | Regression_cos_phi: 0.0407 | Regression_energy: 0.6220 | Regression_eta: 0.0662 | Regression_pt: 0.5862 | Regression_sin_phi: 0.0406 | Group Train Loss Unweighted (window mean): physical_pu: 3.6853 | physical_nopu: 4.5234 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3499 | Classification_binary: 3.3222 | Regression_cos_phi: 1.3204 | Regression_energy: 3.8956 | Regression_eta: 1.1832 | Regression_pt: 5.5113 | Regression_sin_phi: 1.2212 | LR: 2.37e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.5376s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0667 | max_forward: 0.1426 | max_loss: 0.0037 | max_backward: 0.0873 | max_optimizer_step: 0.2591 | max_tensorboard_logging: 1.1650 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5267.4 | uss_MB: 4682.3 | child_rss_MB: 8896.3 | allocated_MB: 394.9 | reserved_MB: 14166.0 | inactive_split_MB: 95.2 +rank4 [2026-10-27 18:54:12,871] INFO : Step 48000/100000 rank4 | Train Loss: 11.2030 | Train Loss Unweighted: 4.3343 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.6447 | Regression_cos_phi: 0.0521 | Regression_energy: 0.2822 | Regression_eta: 0.0657 | Regression_pt: 0.2293 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6844 | physical_nopu: 4.5168 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2603 | Classification_binary: 3.8169 | Regression_cos_phi: 2.1281 | Regression_energy: 1.6729 | Regression_eta: 1.0558 | Regression_pt: 2.0003 | Regression_sin_phi: 2.0093 | LR: 2.31e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.2607s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.0058 | max_backward: 0.0852 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1613 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5253.4 | uss_MB: 4683.3 | child_rss_MB: 8890.9 | allocated_MB: 393.9 | reserved_MB: 14166.0 | inactive_split_MB: 101.2 +rank4 [2026-10-27 18:57:19,782] INFO : Step 49000/100000 rank4 | Train Loss: 12.4372 | Train Loss Unweighted: 4.4483 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.5320 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3881 | Regression_eta: 0.0742 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6801 | physical_nopu: 4.5174 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3714 | Classification_binary: 3.6150 | Regression_cos_phi: 1.7917 | Regression_energy: 2.0212 | Regression_eta: 0.7345 | Regression_pt: 2.6539 | Regression_sin_phi: 1.6441 | LR: 2.26e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.4736s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0009 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1014 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 1.3340 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5240.2 | uss_MB: 4683.4 | child_rss_MB: 8815.5 | allocated_MB: 393.7 | reserved_MB: 14166.0 | inactive_split_MB: 100.8 +rank4 [2026-10-27 19:00:32,233] INFO : Step 50000/100000 rank4 | Train Loss: 13.3654 | Train Loss Unweighted: 4.5742 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4424 | Regression_cos_phi: 0.0426 | Regression_energy: 0.5013 | Regression_eta: 0.0683 | Regression_pt: 0.4573 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6814 | physical_nopu: 4.5132 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3607 | Classification_binary: 3.4008 | Regression_cos_phi: 1.4561 | Regression_energy: 2.3491 | Regression_eta: 0.6397 | Regression_pt: 3.1850 | Regression_sin_phi: 1.6773 | LR: 2.20e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.5862s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0010 | max_forward: 0.1892 | max_loss: 0.0115 | max_backward: 0.1333 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.2641 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0862 | Memory: rss_MB: 5234.2 | uss_MB: 4685.5 | child_rss_MB: 8779.3 | allocated_MB: 394.6 | reserved_MB: 14166.0 | inactive_split_MB: 91.2 +rank4 [2026-10-27 19:00:33,174] INFO : Running validation on rank4 +rank4 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 19:02:37,501] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 19:02:37,940] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 19:03:02,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 19:03:03,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 19:03:03,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 19:03:04,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 19:03:04,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 19:03:05,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 19:03:05,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 19:03:06,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 19:03:16,036] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 19:03:16,262] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 19:03:16,636] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 19:03:31,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 19:03:32,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 19:03:32,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 19:03:32,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 19:03:33,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 19:03:33,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 19:03:34,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 19:03:34,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 19:03:44,728] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 19:03:45,438] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 19:03:45,817] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 19:04:01,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 19:04:01,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 19:04:02,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 19:04:02,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 19:04:03,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 19:04:03,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 19:04:04,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 19:04:04,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 19:04:14,645] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 19:07:44,424] INFO : Step 51000/100000 rank4 | Train Loss: 12.0993 | Train Loss Unweighted: 4.3946 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5485 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3591 | Regression_eta: 0.0789 | Regression_pt: 0.2941 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6745 | physical_nopu: 4.5037 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3433 | Classification_binary: 3.5471 | Regression_cos_phi: 1.8494 | Regression_energy: 1.7594 | Regression_eta: 0.6300 | Regression_pt: 2.4430 | Regression_sin_phi: 1.3319 | LR: 2.14e-04 | DataLoad Time: 0.0098s | Model Forward Time: 3.3976s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0008 | max_forward: 0.1391 | max_loss: 0.0036 | max_backward: 0.1509 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 2.0683 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 7102.6 | uss_MB: 6555.6 | child_rss_MB: 8775.1 | allocated_MB: 393.6 | reserved_MB: 10952.0 | inactive_split_MB: 101.2 +rank4 [2026-10-27 19:10:55,345] INFO : Step 52000/100000 rank4 | Train Loss: 10.7195 | Train Loss Unweighted: 3.6688 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9553 | Regression_cos_phi: 0.0470 | Regression_energy: 0.2719 | Regression_eta: 0.1059 | Regression_pt: 0.2258 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6784 | physical_nopu: 4.5078 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3366 | Classification_binary: 3.0079 | Regression_cos_phi: 1.6465 | Regression_energy: 1.2659 | Regression_eta: 1.2019 | Regression_pt: 1.5499 | Regression_sin_phi: 1.6480 | LR: 2.09e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.5037s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0031 | max_forward: 0.1853 | max_loss: 0.0256 | max_backward: 0.2604 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 2.0239 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7092.6 | uss_MB: 6564.5 | child_rss_MB: 8773.5 | allocated_MB: 404.8 | reserved_MB: 11626.0 | inactive_split_MB: 256.4 +rank4 [2026-10-27 19:13:48,854] INFO : Step 53000/100000 rank4 | Train Loss: 11.7022 | Train Loss Unweighted: 4.3546 | Train Loss Components: Classification: 0.0123 | Classification_binary: 3.5594 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3321 | Regression_eta: 0.0730 | Regression_pt: 0.2795 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6748 | physical_nopu: 4.5005 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2606 | Classification_binary: 3.3680 | Regression_cos_phi: 1.5929 | Regression_energy: 1.3676 | Regression_eta: 0.8379 | Regression_pt: 1.7445 | Regression_sin_phi: 1.3230 | LR: 2.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.1501s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0954 | max_loss: 0.0037 | max_backward: 0.1065 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.9892 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 5255.7 | uss_MB: 4729.4 | child_rss_MB: 8773.6 | allocated_MB: 393.6 | reserved_MB: 11910.0 | inactive_split_MB: 100.6 +rank4 [2026-10-27 19:17:18,773] INFO : Step 54000/100000 rank4 | Train Loss: 12.5762 | Train Loss Unweighted: 4.3665 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0487 | Regression_energy: 0.4104 | Regression_eta: 0.0739 | Regression_pt: 0.3685 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6696 | physical_nopu: 4.5036 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3596 | Classification_binary: 3.3626 | Regression_cos_phi: 1.9563 | Regression_energy: 1.8741 | Regression_eta: 1.0369 | Regression_pt: 2.3601 | Regression_sin_phi: 1.6472 | LR: 1.97e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.3957s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.1022 | max_forward: 0.1953 | max_loss: 0.0037 | max_backward: 0.0975 | max_optimizer_step: 0.2772 | max_tensorboard_logging: 2.0527 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5248.7 | uss_MB: 4730.3 | child_rss_MB: 8744.8 | allocated_MB: 394.9 | reserved_MB: 11910.0 | inactive_split_MB: 118.2 +rank4 [2026-10-27 19:20:28,222] INFO : Step 55000/100000 rank4 | Train Loss: 13.4632 | Train Loss Unweighted: 4.6500 | Train Loss Components: Classification: 0.0134 | Classification_binary: 3.5428 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4921 | Regression_eta: 0.0679 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6720 | physical_nopu: 4.4919 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2182 | Classification_binary: 3.7229 | Regression_cos_phi: 1.9161 | Regression_energy: 1.6480 | Regression_eta: 0.7627 | Regression_pt: 1.8627 | Regression_sin_phi: 1.7809 | LR: 1.92e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.8024s | Phase Times: max_pre_step_barrier: 0.1980 | max_batch_to_device: 0.0007 | max_forward: 0.0889 | max_loss: 0.0037 | max_backward: 0.2091 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 2.3779 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5235.8 | uss_MB: 4729.6 | child_rss_MB: 8640.7 | allocated_MB: 393.8 | reserved_MB: 11910.0 | inactive_split_MB: 89.1 +rank4 [2026-10-27 19:23:30,605] INFO : Step 56000/100000 rank4 | Train Loss: 11.6078 | Train Loss Unweighted: 4.2962 | Train Loss Components: Classification: 0.0140 | Classification_binary: 3.4759 | Regression_cos_phi: 0.0438 | Regression_energy: 0.3484 | Regression_eta: 0.0671 | Regression_pt: 0.2977 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6692 | physical_nopu: 4.4908 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2969 | Classification_binary: 3.5874 | Regression_cos_phi: 1.4154 | Regression_energy: 1.2924 | Regression_eta: 0.6232 | Regression_pt: 1.6319 | Regression_sin_phi: 1.3225 | LR: 1.86e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.2295s | Phase Times: max_pre_step_barrier: 0.0941 | max_batch_to_device: 0.0008 | max_forward: 0.0930 | max_loss: 0.0038 | max_backward: 0.0988 | max_optimizer_step: 0.1756 | max_tensorboard_logging: 1.8920 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 5221.8 | uss_MB: 4732.9 | child_rss_MB: 8582.9 | allocated_MB: 393.5 | reserved_MB: 11910.0 | inactive_split_MB: 115.7 +rank4 [2026-10-27 19:26:52,902] INFO : Step 57000/100000 rank4 | Train Loss: 13.9252 | Train Loss Unweighted: 4.6468 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4640 | Regression_cos_phi: 0.0464 | Regression_energy: 0.5144 | Regression_eta: 0.0781 | Regression_pt: 0.4791 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6687 | physical_nopu: 4.4875 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3686 | Classification_binary: 3.4964 | Regression_cos_phi: 1.5670 | Regression_energy: 2.9332 | Regression_eta: 0.6661 | Regression_pt: 4.2447 | Regression_sin_phi: 1.8817 | LR: 1.81e-04 | DataLoad Time: 0.0021s | Model Forward Time: 4.6264s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.1223 | max_forward: 0.2094 | max_loss: 0.0420 | max_backward: 0.1628 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 2.9072 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 5216.8 | uss_MB: 4734.2 | child_rss_MB: 8511.2 | allocated_MB: 394.3 | reserved_MB: 12240.0 | inactive_split_MB: 101.4 +rank4 [2026-10-27 19:30:07,652] INFO : Step 58000/100000 rank4 | Train Loss: 11.5910 | Train Loss Unweighted: 4.3047 | Train Loss Components: Classification: 0.0106 | Classification_binary: 3.5490 | Regression_cos_phi: 0.0544 | Regression_energy: 0.3108 | Regression_eta: 0.0631 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0510 | Group Train Loss Unweighted (window mean): physical_pu: 3.6630 | physical_nopu: 4.4842 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2550 | Classification_binary: 3.7118 | Regression_cos_phi: 1.8063 | Regression_energy: 1.3056 | Regression_eta: 0.7161 | Regression_pt: 1.5739 | Regression_sin_phi: 2.0302 | LR: 1.75e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.5120s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.1080 | max_backward: 0.1964 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.2749 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 5209.3 | uss_MB: 4734.4 | child_rss_MB: 8432.6 | allocated_MB: 393.9 | reserved_MB: 12240.0 | inactive_split_MB: 121.9 +rank4 [2026-10-27 19:33:12,149] INFO : Step 59000/100000 rank4 | Train Loss: 12.2218 | Train Loss Unweighted: 4.4115 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5166 | Regression_cos_phi: 0.0458 | Regression_energy: 0.3794 | Regression_eta: 0.0771 | Regression_pt: 0.3280 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6650 | physical_nopu: 4.4788 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3203 | Classification_binary: 3.3699 | Regression_cos_phi: 1.4254 | Regression_energy: 1.4765 | Regression_eta: 0.6576 | Regression_pt: 1.9534 | Regression_sin_phi: 1.5357 | LR: 1.70e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.2208s | Phase Times: max_pre_step_barrier: 0.6889 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0499 | max_backward: 0.1549 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5201.3 | uss_MB: 4735.4 | child_rss_MB: 8387.3 | allocated_MB: 394.0 | reserved_MB: 12240.0 | inactive_split_MB: 135.3 +rank4 [2026-10-27 19:36:26,517] INFO : Step 60000/100000 rank4 | Train Loss: 14.3810 | Train Loss Unweighted: 4.7229 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.4146 | Regression_cos_phi: 0.0449 | Regression_energy: 0.5841 | Regression_eta: 0.0716 | Regression_pt: 0.5495 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.6604 | physical_nopu: 4.4865 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3293 | Classification_binary: 3.3013 | Regression_cos_phi: 1.4045 | Regression_energy: 2.1085 | Regression_eta: 0.6502 | Regression_pt: 2.8926 | Regression_sin_phi: 1.2793 | LR: 1.64e-04 | DataLoad Time: 0.0030s | Model Forward Time: 2.4561s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0849 | max_loss: 0.0036 | max_backward: 0.1478 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 1.2880 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5191.9 | uss_MB: 4495.0 | child_rss_MB: 8328.9 | allocated_MB: 394.8 | reserved_MB: 12240.0 | inactive_split_MB: 87.0 +rank4 [2026-10-27 19:36:27,558] INFO : Running validation on rank4 +rank4 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 19:38:18,600] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 19:38:19,028] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 19:38:43,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 19:38:44,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 19:38:44,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 19:38:45,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 19:38:45,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 19:38:46,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 19:38:46,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 19:38:47,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 19:38:57,046] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 19:38:57,241] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 19:38:57,610] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 19:39:12,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 19:39:13,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 19:39:13,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 19:39:13,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 19:39:14,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 19:39:14,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 19:39:15,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 19:39:15,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 19:39:25,754] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 19:39:26,170] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 19:39:26,533] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 19:39:41,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 19:39:42,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 19:39:42,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 19:39:43,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 19:39:43,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 19:39:43,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 19:39:44,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 19:39:44,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 19:39:54,847] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 19:43:33,544] INFO : Step 61000/100000 rank4 | Train Loss: 11.4262 | Train Loss Unweighted: 4.3706 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.6500 | Regression_cos_phi: 0.0531 | Regression_energy: 0.2956 | Regression_eta: 0.0675 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6553 | physical_nopu: 4.4780 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2093 | Classification_binary: 3.6238 | Regression_cos_phi: 1.6365 | Regression_energy: 1.5900 | Regression_eta: 0.8094 | Regression_pt: 2.0910 | Regression_sin_phi: 1.7076 | LR: 1.59e-04 | DataLoad Time: 0.0076s | Model Forward Time: 3.1987s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1116 | max_loss: 0.0036 | max_backward: 0.1506 | max_optimizer_step: 0.1828 | max_tensorboard_logging: 1.9738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 5784.0 | uss_MB: 5329.2 | child_rss_MB: 8302.4 | allocated_MB: 393.5 | reserved_MB: 11190.0 | inactive_split_MB: 103.1 +rank4 [2026-10-27 19:46:49,589] INFO : Step 62000/100000 rank4 | Train Loss: 10.7781 | Train Loss Unweighted: 3.6510 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9253 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2791 | Regression_eta: 0.1043 | Regression_pt: 0.2316 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3221 | Classification_binary: 2.9160 | Regression_cos_phi: 1.6478 | Regression_energy: 1.3429 | Regression_eta: 1.1630 | Regression_pt: 1.6533 | Regression_sin_phi: 1.5860 | LR: 1.54e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.6558s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0030 | max_forward: 0.1294 | max_loss: 0.0038 | max_backward: 0.2007 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.2115 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0496 | Memory: rss_MB: 5775.0 | uss_MB: 5052.2 | child_rss_MB: 8186.6 | allocated_MB: 404.4 | reserved_MB: 11190.0 | inactive_split_MB: 148.6 +rank4 [2026-10-27 19:50:02,443] INFO : Step 63000/100000 rank4 | Train Loss: 11.5766 | Train Loss Unweighted: 4.3655 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6000 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3204 | Regression_eta: 0.0727 | Regression_pt: 0.2610 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6589 | physical_nopu: 4.4771 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2952 | Classification_binary: 3.6519 | Regression_cos_phi: 2.0265 | Regression_energy: 2.1556 | Regression_eta: 1.1438 | Regression_pt: 2.5141 | Regression_sin_phi: 1.6572 | LR: 1.49e-04 | DataLoad Time: 0.0011s | Model Forward Time: 2.3902s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.1036 | max_loss: 0.0037 | max_backward: 0.1140 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2208 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 5773.0 | uss_MB: 5331.6 | child_rss_MB: 8138.8 | allocated_MB: 393.8 | reserved_MB: 11190.0 | inactive_split_MB: 93.0 +rank4 [2026-10-27 19:53:17,771] INFO : Step 64000/100000 rank4 | Train Loss: 13.2709 | Train Loss Unweighted: 4.5542 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.4295 | Regression_cos_phi: 0.0429 | Regression_energy: 0.4948 | Regression_eta: 0.0672 | Regression_pt: 0.4587 | Regression_sin_phi: 0.0436 | Group Train Loss Unweighted (window mean): physical_pu: 3.6551 | physical_nopu: 4.4714 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4111 | Classification_binary: 3.3855 | Regression_cos_phi: 1.4625 | Regression_energy: 2.9104 | Regression_eta: 1.3389 | Regression_pt: 3.8699 | Regression_sin_phi: 1.4395 | LR: 1.43e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.3963s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0928 | max_loss: 0.0570 | max_backward: 0.1589 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 1.2035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 5289.6 | uss_MB: 4857.6 | child_rss_MB: 8120.4 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank4 [2026-10-27 19:56:23,053] INFO : Step 65000/100000 rank4 | Train Loss: 15.6652 | Train Loss Unweighted: 4.9833 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5112 | Regression_cos_phi: 0.0496 | Regression_energy: 0.6688 | Regression_eta: 0.0629 | Regression_pt: 0.6290 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6554 | physical_nopu: 4.4650 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2892 | Classification_binary: 3.3425 | Regression_cos_phi: 1.6681 | Regression_energy: 1.7331 | Regression_eta: 0.7031 | Regression_pt: 2.2908 | Regression_sin_phi: 1.2503 | LR: 1.38e-04 | DataLoad Time: 0.0014s | Model Forward Time: 2.7634s | Phase Times: max_pre_step_barrier: 0.1158 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0042 | max_backward: 0.1974 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.4176 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5282.6 | uss_MB: 4858.3 | child_rss_MB: 8025.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 124.9 +rank4 [2026-10-27 19:59:28,722] INFO : Step 66000/100000 rank4 | Train Loss: 12.0073 | Train Loss Unweighted: 4.3350 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4949 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3535 | Regression_eta: 0.0691 | Regression_pt: 0.3018 | Regression_sin_phi: 0.0501 | Group Train Loss Unweighted (window mean): physical_pu: 3.6526 | physical_nopu: 4.4657 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3519 | Classification_binary: 3.4868 | Regression_cos_phi: 1.5401 | Regression_energy: 1.5586 | Regression_eta: 0.5577 | Regression_pt: 1.9999 | Regression_sin_phi: 1.6737 | LR: 1.33e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.4468s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0891 | max_loss: 0.0538 | max_backward: 0.1468 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 1.2753 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5277.7 | uss_MB: 4858.9 | child_rss_MB: 7982.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 107.1 +rank4 [2026-10-27 20:02:51,910] INFO : Step 67000/100000 rank4 | Train Loss: 13.3177 | Train Loss Unweighted: 4.5554 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.4800 | Regression_cos_phi: 0.0500 | Regression_energy: 0.4690 | Regression_eta: 0.0673 | Regression_pt: 0.4253 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6538 | physical_nopu: 4.4665 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3583 | Classification_binary: 3.4274 | Regression_cos_phi: 1.2955 | Regression_energy: 2.2466 | Regression_eta: 0.5182 | Regression_pt: 3.0110 | Regression_sin_phi: 1.4997 | LR: 1.28e-04 | DataLoad Time: 0.0008s | Model Forward Time: 3.9390s | Phase Times: max_pre_step_barrier: 0.0792 | max_batch_to_device: 0.0009 | max_forward: 0.0886 | max_loss: 0.1057 | max_backward: 0.1987 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 2.6109 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 5271.7 | uss_MB: 4859.8 | child_rss_MB: 7952.8 | allocated_MB: 393.9 | reserved_MB: 11792.0 | inactive_split_MB: 111.5 +rank4 [2026-10-27 20:06:09,103] INFO : Step 68000/100000 rank4 | Train Loss: 11.1535 | Train Loss Unweighted: 4.2643 | Train Loss Components: Classification: 0.0109 | Classification_binary: 3.5138 | Regression_cos_phi: 0.0456 | Regression_energy: 0.3163 | Regression_eta: 0.0648 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6527 | physical_nopu: 4.4693 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1799 | Classification_binary: 3.4262 | Regression_cos_phi: 1.7707 | Regression_energy: 1.7000 | Regression_eta: 0.7071 | Regression_pt: 2.0194 | Regression_sin_phi: 1.5249 | LR: 1.24e-04 | DataLoad Time: 0.0014s | Model Forward Time: 2.3904s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1008 | max_loss: 0.0767 | max_backward: 0.1768 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.1759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5263.4 | uss_MB: 4860.3 | child_rss_MB: 7883.7 | allocated_MB: 394.5 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank4 [2026-10-27 20:09:12,492] INFO : Step 69000/100000 rank4 | Train Loss: 12.1728 | Train Loss Unweighted: 4.3418 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4719 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3660 | Regression_eta: 0.0763 | Regression_pt: 0.3115 | Regression_sin_phi: 0.0548 | Group Train Loss Unweighted (window mean): physical_pu: 3.6494 | physical_nopu: 4.4629 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3397 | Classification_binary: 3.4547 | Regression_cos_phi: 1.2937 | Regression_energy: 1.6401 | Regression_eta: 0.4931 | Regression_pt: 2.1556 | Regression_sin_phi: 1.9433 | LR: 1.19e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.2957s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.1094 | max_loss: 0.0147 | max_backward: 0.1167 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 1.1204 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5251.4 | uss_MB: 4861.5 | child_rss_MB: 7816.8 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 106.5 +rank4 [2026-10-27 20:12:13,498] INFO : Step 70000/100000 rank4 | Train Loss: 14.9156 | Train Loss Unweighted: 4.7730 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.3975 | Regression_cos_phi: 0.0487 | Regression_energy: 0.6183 | Regression_eta: 0.0638 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0448 | Group Train Loss Unweighted (window mean): physical_pu: 3.6488 | physical_nopu: 4.4584 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3896 | Classification_binary: 3.3326 | Regression_cos_phi: 1.6843 | Regression_energy: 3.6278 | Regression_eta: 0.6637 | Regression_pt: 5.2712 | Regression_sin_phi: 1.5145 | LR: 1.14e-04 | DataLoad Time: 0.0068s | Model Forward Time: 2.3868s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1038 | max_loss: 0.0858 | max_backward: 0.1871 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 1.1619 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5245.8 | uss_MB: 4860.8 | child_rss_MB: 7775.8 | allocated_MB: 395.0 | reserved_MB: 11792.0 | inactive_split_MB: 111.9 +rank4 [2026-10-27 20:12:14,505] INFO : Running validation on rank4 +rank4 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 20:14:09,431] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 20:14:09,864] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 20:14:34,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 20:14:35,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 20:14:35,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 20:14:36,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 20:14:36,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 20:14:36,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 20:14:37,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 20:14:37,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 20:14:47,926] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 20:14:47,942] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 20:14:48,315] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 20:15:03,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 20:15:03,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 20:15:04,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 20:15:04,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 20:15:05,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 20:15:05,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 20:15:06,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 20:15:06,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 20:15:16,487] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 20:15:17,039] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 20:15:17,439] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 20:15:32,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 20:15:33,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 20:15:33,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 20:15:34,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 20:15:34,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 20:15:35,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 20:15:35,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 20:15:36,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 20:15:46,169] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 20:19:21,751] INFO : Step 71000/100000 rank4 | Train Loss: 10.9922 | Train Loss Unweighted: 3.6772 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9201 | Regression_cos_phi: 0.0477 | Regression_energy: 0.2906 | Regression_eta: 0.1066 | Regression_pt: 0.2485 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6484 | physical_nopu: 4.4561 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3690 | Classification_binary: 2.9706 | Regression_cos_phi: 1.7056 | Regression_energy: 1.3884 | Regression_eta: 1.1888 | Regression_pt: 1.7601 | Regression_sin_phi: 1.6035 | LR: 1.10e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.5360s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0652 | max_forward: 0.2596 | max_loss: 0.0036 | max_backward: 0.3497 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 1.8772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 7279.8 | uss_MB: 6900.2 | child_rss_MB: 7804.4 | allocated_MB: 405.9 | reserved_MB: 11130.0 | inactive_split_MB: 135.5 +rank4 [2026-10-27 20:22:34,778] INFO : Step 72000/100000 rank4 | Train Loss: 14.4733 | Train Loss Unweighted: 4.6898 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3667 | Regression_cos_phi: 0.0458 | Regression_energy: 0.5948 | Regression_eta: 0.0650 | Regression_pt: 0.5567 | Regression_sin_phi: 0.0444 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4536 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3808 | Classification_binary: 3.3231 | Regression_cos_phi: 1.6161 | Regression_energy: 2.7912 | Regression_eta: 0.7892 | Regression_pt: 3.7880 | Regression_sin_phi: 1.4493 | LR: 1.05e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.5070s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1390 | max_loss: 0.0037 | max_backward: 0.1007 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.2476 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1281 | Memory: rss_MB: 5316.2 | uss_MB: 4936.1 | child_rss_MB: 7727.0 | allocated_MB: 393.8 | reserved_MB: 11706.0 | inactive_split_MB: 88.5 +rank4 [2026-10-27 20:25:34,428] INFO : Step 73000/100000 rank4 | Train Loss: 11.7903 | Train Loss Unweighted: 4.2561 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.5010 | Regression_cos_phi: 0.0578 | Regression_energy: 0.3092 | Regression_eta: 0.0644 | Regression_pt: 0.2565 | Regression_sin_phi: 0.0557 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4567 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2592 | Classification_binary: 3.5460 | Regression_cos_phi: 2.1342 | Regression_energy: 1.6830 | Regression_eta: 0.5854 | Regression_pt: 2.0743 | Regression_sin_phi: 1.9351 | LR: 1.01e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.6068s | Phase Times: max_pre_step_barrier: 0.1246 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0122 | max_backward: 0.1972 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.2348 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5307.2 | uss_MB: 4937.3 | child_rss_MB: 7690.0 | allocated_MB: 393.5 | reserved_MB: 11706.0 | inactive_split_MB: 93.2 +rank4 [2026-10-27 20:28:47,370] INFO : Step 74000/100000 rank4 | Train Loss: 11.6748 | Train Loss Unweighted: 4.2803 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4309 | Regression_cos_phi: 0.0428 | Regression_energy: 0.3572 | Regression_eta: 0.0695 | Regression_pt: 0.3177 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.6485 | physical_nopu: 4.4502 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3919 | Classification_binary: 3.5420 | Regression_cos_phi: 1.4507 | Regression_energy: 1.6501 | Regression_eta: 0.7389 | Regression_pt: 2.1650 | Regression_sin_phi: 1.4978 | LR: 9.68e-05 | DataLoad Time: 0.0010s | Model Forward Time: 3.0492s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1195 | max_loss: 0.0060 | max_backward: 0.2082 | max_optimizer_step: 0.1763 | max_tensorboard_logging: 1.8023 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5300.9 | uss_MB: 4938.3 | child_rss_MB: 7614.5 | allocated_MB: 394.3 | reserved_MB: 11706.0 | inactive_split_MB: 97.0 +rank4 [2026-10-27 20:31:56,455] INFO : Step 75000/100000 rank4 | Train Loss: 14.1604 | Train Loss Unweighted: 4.6445 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.4007 | Regression_cos_phi: 0.0489 | Regression_energy: 0.5577 | Regression_eta: 0.0640 | Regression_pt: 0.5112 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.6489 | physical_nopu: 4.4484 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4105 | Classification_binary: 3.3152 | Regression_cos_phi: 1.5936 | Regression_energy: 3.2163 | Regression_eta: 0.7112 | Regression_pt: 4.3256 | Regression_sin_phi: 1.2428 | LR: 9.27e-05 | DataLoad Time: 0.0025s | Model Forward Time: 2.9327s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0831 | max_loss: 0.0038 | max_backward: 0.2058 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.5751 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1279 | Memory: rss_MB: 5290.9 | uss_MB: 4938.0 | child_rss_MB: 7581.3 | allocated_MB: 394.4 | reserved_MB: 11706.0 | inactive_split_MB: 89.1 +rank4 [2026-10-27 20:34:57,733] INFO : Step 76000/100000 rank4 | Train Loss: 11.4149 | Train Loss Unweighted: 4.2984 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5224 | Regression_cos_phi: 0.0478 | Regression_energy: 0.3298 | Regression_eta: 0.0709 | Regression_pt: 0.2685 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6454 | physical_nopu: 4.4486 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2309 | Classification_binary: 3.5814 | Regression_cos_phi: 1.7137 | Regression_energy: 1.1643 | Regression_eta: 0.6087 | Regression_pt: 1.5286 | Regression_sin_phi: 1.3512 | LR: 8.88e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.1729s | Phase Times: max_pre_step_barrier: 0.0296 | max_batch_to_device: 0.0996 | max_forward: 0.1816 | max_loss: 0.0036 | max_backward: 0.0930 | max_optimizer_step: 0.2829 | max_tensorboard_logging: 1.7104 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1287 | Memory: rss_MB: 5286.0 | uss_MB: 4937.3 | child_rss_MB: 7586.8 | allocated_MB: 393.6 | reserved_MB: 12330.0 | inactive_split_MB: 97.7 +rank4 [2026-10-27 20:38:16,884] INFO : Step 77000/100000 rank4 | Train Loss: 12.8789 | Train Loss Unweighted: 4.4653 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4333 | Regression_cos_phi: 0.0478 | Regression_energy: 0.4517 | Regression_eta: 0.0624 | Regression_pt: 0.4079 | Regression_sin_phi: 0.0461 | Group Train Loss Unweighted (window mean): physical_pu: 3.6433 | physical_nopu: 4.4448 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3712 | Classification_binary: 3.7457 | Regression_cos_phi: 1.9657 | Regression_energy: 1.6629 | Regression_eta: 0.7208 | Regression_pt: 2.2898 | Regression_sin_phi: 1.5827 | LR: 8.50e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.0361s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.0831 | max_loss: 0.0514 | max_backward: 0.1524 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.8652 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5273.0 | uss_MB: 4939.8 | child_rss_MB: 7463.9 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 99.2 +rank4 [2026-10-27 20:41:21,661] INFO : Step 78000/100000 rank4 | Train Loss: 13.2420 | Train Loss Unweighted: 4.6345 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.5815 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4598 | Regression_eta: 0.0679 | Regression_pt: 0.4144 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6415 | physical_nopu: 4.4494 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2902 | Classification_binary: 3.5693 | Regression_cos_phi: 1.6054 | Regression_energy: 1.7224 | Regression_eta: 0.8262 | Regression_pt: 2.2535 | Regression_sin_phi: 1.6062 | LR: 8.13e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.0716s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0927 | max_loss: 0.0036 | max_backward: 0.2060 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.8450 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5261.0 | uss_MB: 4940.8 | child_rss_MB: 7383.4 | allocated_MB: 394.7 | reserved_MB: 12330.0 | inactive_split_MB: 95.1 +rank4 [2026-10-27 20:44:31,405] INFO : Step 79000/100000 rank4 | Train Loss: 11.5047 | Train Loss Unweighted: 4.2761 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4919 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3261 | Regression_eta: 0.0640 | Regression_pt: 0.2834 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6416 | physical_nopu: 4.4460 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3137 | Classification_binary: 3.4801 | Regression_cos_phi: 2.0197 | Regression_energy: 2.1249 | Regression_eta: 0.7613 | Regression_pt: 2.6719 | Regression_sin_phi: 2.4238 | LR: 7.78e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.2159s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0253 | max_forward: 0.1473 | max_loss: 0.0038 | max_backward: 0.0955 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 1.9569 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5248.8 | uss_MB: 4513.6 | child_rss_MB: 7321.5 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 120.0 +rank4 [2026-10-27 20:47:43,818] INFO : Step 80000/100000 rank4 | Train Loss: 15.3065 | Train Loss Unweighted: 4.8564 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4208 | Regression_cos_phi: 0.0455 | Regression_energy: 0.6415 | Regression_eta: 0.0773 | Regression_pt: 0.6092 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6398 | physical_nopu: 4.4406 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3893 | Classification_binary: 3.4983 | Regression_cos_phi: 1.8507 | Regression_energy: 2.4462 | Regression_eta: 0.6957 | Regression_pt: 3.4228 | Regression_sin_phi: 1.7149 | LR: 7.44e-05 | DataLoad Time: 0.0091s | Model Forward Time: 3.0633s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0010 | max_forward: 0.0840 | max_loss: 0.0039 | max_backward: 0.0916 | max_optimizer_step: 0.2630 | max_tensorboard_logging: 1.7339 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5238.8 | uss_MB: 4942.0 | child_rss_MB: 7231.1 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 141.7 +rank4 [2026-10-27 20:47:44,754] INFO : Running validation on rank4 +rank4 [2026-10-27 20:47:44,754] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 20:47:44,754] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 20:47:44,755] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 20:47:44,755] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 20:49:38,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 20:49:38,983] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 20:50:04,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 20:50:05,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 20:50:05,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 20:50:06,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 20:50:06,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 20:50:07,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 20:50:07,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 20:50:08,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 20:50:18,128] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 20:50:18,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 20:50:18,746] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 20:50:33,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 20:50:34,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 20:50:34,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 20:50:34,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 20:50:35,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 20:50:35,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 20:50:36,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 20:50:36,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 20:50:46,765] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 20:50:47,267] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 20:50:47,676] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 20:51:02,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 20:51:03,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 20:51:03,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 20:51:04,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 20:51:04,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 20:51:04,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 20:51:05,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 20:51:05,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 20:51:15,854] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 20:54:47,405] INFO : Step 81000/100000 rank4 | Train Loss: 10.7072 | Train Loss Unweighted: 3.6241 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.9080 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2710 | Regression_eta: 0.1078 | Regression_pt: 0.2269 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6413 | physical_nopu: 4.4395 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3741 | Classification_binary: 2.9896 | Regression_cos_phi: 1.7225 | Regression_energy: 1.3063 | Regression_eta: 1.1603 | Regression_pt: 1.6157 | Regression_sin_phi: 1.6580 | LR: 7.11e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.4726s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0033 | max_forward: 0.1984 | max_loss: 0.0038 | max_backward: 0.3244 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 1.8991 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 7314.5 | uss_MB: 7027.3 | child_rss_MB: 7215.1 | allocated_MB: 408.0 | reserved_MB: 11650.0 | inactive_split_MB: 261.6 +rank4 [2026-10-27 20:58:04,709] INFO : Step 82000/100000 rank4 | Train Loss: 12.9094 | Train Loss Unweighted: 4.6043 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5247 | Regression_cos_phi: 0.0417 | Regression_energy: 0.4765 | Regression_eta: 0.0603 | Regression_pt: 0.4449 | Regression_sin_phi: 0.0402 | Group Train Loss Unweighted (window mean): physical_pu: 3.6409 | physical_nopu: 4.4420 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2994 | Classification_binary: 3.6522 | Regression_cos_phi: 1.5602 | Regression_energy: 1.3970 | Regression_eta: 0.5231 | Regression_pt: 1.7671 | Regression_sin_phi: 1.5816 | LR: 6.80e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.1711s | Phase Times: max_pre_step_barrier: 0.0156 | max_batch_to_device: 0.0008 | max_forward: 0.1472 | max_loss: 0.0038 | max_backward: 0.1553 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 1.8498 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5342.3 | uss_MB: 5059.9 | child_rss_MB: 7159.0 | allocated_MB: 393.8 | reserved_MB: 11650.0 | inactive_split_MB: 92.9 +rank4 [2026-10-27 21:01:20,554] INFO : Step 83000/100000 rank4 | Train Loss: 11.2556 | Train Loss Unweighted: 4.2618 | Train Loss Components: Classification: 0.0094 | Classification_binary: 3.5167 | Regression_cos_phi: 0.0501 | Regression_energy: 0.3127 | Regression_eta: 0.0669 | Regression_pt: 0.2586 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6408 | physical_nopu: 4.4397 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2083 | Classification_binary: 3.4967 | Regression_cos_phi: 1.6495 | Regression_energy: 1.4275 | Regression_eta: 0.7518 | Regression_pt: 1.7507 | Regression_sin_phi: 1.4248 | LR: 6.51e-05 | DataLoad Time: 0.0018s | Model Forward Time: 2.5753s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0008 | max_forward: 0.1683 | max_loss: 0.0037 | max_backward: 0.1023 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.3519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 5336.3 | uss_MB: 5061.6 | child_rss_MB: 7095.7 | allocated_MB: 393.2 | reserved_MB: 11650.0 | inactive_split_MB: 95.2 +rank4 [2026-10-27 21:04:33,664] INFO : Step 84000/100000 rank4 | Train Loss: 11.5964 | Train Loss Unweighted: 4.2611 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4346 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3486 | Regression_eta: 0.0638 | Regression_pt: 0.3055 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6366 | physical_nopu: 4.4419 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3950 | Classification_binary: 3.4117 | Regression_cos_phi: 1.6659 | Regression_energy: 1.4863 | Regression_eta: 0.8046 | Regression_pt: 1.8745 | Regression_sin_phi: 1.4356 | LR: 6.23e-05 | DataLoad Time: 0.1564s | Model Forward Time: 4.7793s | Phase Times: max_pre_step_barrier: 0.2933 | max_batch_to_device: 0.0008 | max_forward: 0.1129 | max_loss: 0.0110 | max_backward: 0.1312 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 3.4426 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5330.3 | uss_MB: 5063.1 | child_rss_MB: 7029.9 | allocated_MB: 393.9 | reserved_MB: 11650.0 | inactive_split_MB: 95.6 +rank4 [2026-10-27 21:07:28,408] INFO : Step 85000/100000 rank4 | Train Loss: 14.1090 | Train Loss Unweighted: 4.6641 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.3775 | Regression_cos_phi: 0.0425 | Regression_energy: 0.5754 | Regression_eta: 0.0615 | Regression_pt: 0.5479 | Regression_sin_phi: 0.0420 | Group Train Loss Unweighted (window mean): physical_pu: 3.6325 | physical_nopu: 4.4336 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3966 | Classification_binary: 3.1597 | Regression_cos_phi: 1.4201 | Regression_energy: 3.7702 | Regression_eta: 0.5202 | Regression_pt: 5.3932 | Regression_sin_phi: 1.6212 | LR: 5.96e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.5393s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0496 | max_backward: 0.1488 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 1.3583 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 5323.1 | uss_MB: 5063.4 | child_rss_MB: 7037.3 | allocated_MB: 394.0 | reserved_MB: 11698.0 | inactive_split_MB: 93.1 +rank4 [2026-10-27 21:10:26,799] INFO : Step 86000/100000 rank4 | Train Loss: 11.7921 | Train Loss Unweighted: 4.3412 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.5823 | Regression_cos_phi: 0.0537 | Regression_energy: 0.3118 | Regression_eta: 0.0750 | Regression_pt: 0.2521 | Regression_sin_phi: 0.0550 | Group Train Loss Unweighted (window mean): physical_pu: 3.6404 | physical_nopu: 4.4337 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2143 | Classification_binary: 3.6399 | Regression_cos_phi: 1.7474 | Regression_energy: 1.5680 | Regression_eta: 0.7701 | Regression_pt: 1.7331 | Regression_sin_phi: 2.2641 | LR: 5.71e-05 | DataLoad Time: 0.0020s | Model Forward Time: 2.4340s | Phase Times: max_pre_step_barrier: 0.0233 | max_batch_to_device: 0.0898 | max_forward: 0.1824 | max_loss: 0.0038 | max_backward: 0.0938 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.9995 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1323 | Memory: rss_MB: 5319.1 | uss_MB: 4616.7 | child_rss_MB: 7006.7 | allocated_MB: 393.2 | reserved_MB: 11704.0 | inactive_split_MB: 96.0 +rank4 [2026-10-27 21:13:31,178] INFO : Step 87000/100000 rank4 | Train Loss: 12.0054 | Train Loss Unweighted: 4.3232 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.3964 | Regression_cos_phi: 0.0444 | Regression_energy: 0.4046 | Regression_eta: 0.0620 | Regression_pt: 0.3567 | Regression_sin_phi: 0.0418 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3627 | Classification_binary: 3.4449 | Regression_cos_phi: 1.5139 | Regression_energy: 1.6523 | Regression_eta: 0.6225 | Regression_pt: 2.1248 | Regression_sin_phi: 1.4526 | LR: 5.48e-05 | DataLoad Time: 0.0017s | Model Forward Time: 2.3326s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.1290 | max_loss: 0.0039 | max_backward: 0.0975 | max_optimizer_step: 0.1784 | max_tensorboard_logging: 1.1620 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5310.9 | uss_MB: 5064.7 | child_rss_MB: 7011.5 | allocated_MB: 394.7 | reserved_MB: 11704.0 | inactive_split_MB: 94.3 +rank4 [2026-10-27 21:16:33,434] INFO : Step 88000/100000 rank4 | Train Loss: 14.2663 | Train Loss Unweighted: 4.6673 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.4533 | Regression_cos_phi: 0.0523 | Regression_energy: 0.5342 | Regression_eta: 0.0666 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0508 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3848 | Classification_binary: 3.6866 | Regression_cos_phi: 1.9514 | Regression_energy: 3.2614 | Regression_eta: 1.0652 | Regression_pt: 4.6035 | Regression_sin_phi: 1.7941 | LR: 5.26e-05 | DataLoad Time: 0.0023s | Model Forward Time: 2.3217s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0008 | max_forward: 0.0997 | max_loss: 0.0039 | max_backward: 0.1038 | max_optimizer_step: 0.2517 | max_tensorboard_logging: 1.0175 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5309.9 | uss_MB: 5065.2 | child_rss_MB: 7012.4 | allocated_MB: 393.7 | reserved_MB: 11704.0 | inactive_split_MB: 103.2 +rank4 [2026-10-27 21:19:30,808] INFO : Step 89000/100000 rank4 | Train Loss: 11.9885 | Train Loss Unweighted: 4.3122 | Train Loss Components: Classification: 0.0138 | Classification_binary: 3.4837 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3438 | Regression_eta: 0.0687 | Regression_pt: 0.2984 | Regression_sin_phi: 0.0554 | Group Train Loss Unweighted (window mean): physical_pu: 3.6329 | physical_nopu: 4.4374 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2951 | Classification_binary: 3.5613 | Regression_cos_phi: 1.6710 | Regression_energy: 1.7741 | Regression_eta: 1.1674 | Regression_pt: 2.1143 | Regression_sin_phi: 2.1615 | LR: 5.06e-05 | DataLoad Time: 0.0017s | Model Forward Time: 2.2339s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0976 | max_loss: 0.0053 | max_backward: 0.1242 | max_optimizer_step: 0.1785 | max_tensorboard_logging: 1.0326 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 5307.5 | uss_MB: 5066.4 | child_rss_MB: 7006.7 | allocated_MB: 395.0 | reserved_MB: 13422.0 | inactive_split_MB: 109.9 +rank4 [2026-10-27 21:22:22,522] INFO : Step 90000/100000 rank4 | Train Loss: 10.5888 | Train Loss Unweighted: 3.6256 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9282 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2656 | Regression_eta: 0.1044 | Regression_pt: 0.2178 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6354 | physical_nopu: 4.4354 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3047 | Classification_binary: 2.9383 | Regression_cos_phi: 1.6907 | Regression_energy: 1.2546 | Regression_eta: 1.1475 | Regression_pt: 1.4980 | Regression_sin_phi: 1.6330 | LR: 4.88e-05 | DataLoad Time: 0.0019s | Model Forward Time: 4.1186s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0033 | max_forward: 0.1953 | max_loss: 0.0645 | max_backward: 0.8258 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.9975 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5307.5 | uss_MB: 5074.7 | child_rss_MB: 7021.2 | allocated_MB: 404.6 | reserved_MB: 13422.0 | inactive_split_MB: 208.4 +rank4 [2026-10-27 21:22:23,507] INFO : Running validation on rank4 +rank4 [2026-10-27 21:22:23,507] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 21:22:23,507] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 21:22:23,507] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 21:22:23,507] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 21:24:23,625] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 21:24:24,116] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 21:24:48,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 21:24:49,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 21:24:49,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 21:24:50,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 21:24:50,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 21:24:51,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 21:24:51,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 21:24:52,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 21:25:02,174] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 21:25:02,472] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 21:25:02,869] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 21:25:18,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 21:25:19,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 21:25:19,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 21:25:19,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 21:25:20,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 21:25:20,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 21:25:21,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 21:25:21,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 21:25:31,774] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 21:25:32,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 21:25:32,492] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 21:25:47,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 21:25:48,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 21:25:48,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 21:25:49,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 21:25:49,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 21:25:50,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 21:25:50,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 21:25:51,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 21:26:01,154] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 21:29:48,334] INFO : Step 91000/100000 rank4 | Train Loss: 11.9808 | Train Loss Unweighted: 4.3520 | Train Loss Components: Classification: 0.0128 | Classification_binary: 3.5260 | Regression_cos_phi: 0.0522 | Regression_energy: 0.3480 | Regression_eta: 0.0723 | Regression_pt: 0.2901 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4346 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2752 | Classification_binary: 3.5396 | Regression_cos_phi: 2.1278 | Regression_energy: 1.6094 | Regression_eta: 0.6882 | Regression_pt: 1.9189 | Regression_sin_phi: 2.1238 | LR: 4.71e-05 | DataLoad Time: 0.0015s | Model Forward Time: 3.6020s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1743 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.3267 | max_tensorboard_logging: 2.0733 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0716 | Memory: rss_MB: 7308.2 | uss_MB: 7066.1 | child_rss_MB: 7044.9 | allocated_MB: 393.7 | reserved_MB: 11392.0 | inactive_split_MB: 89.1 +rank4 [2026-10-27 21:33:08,656] INFO : Step 92000/100000 rank4 | Train Loss: 14.4974 | Train Loss Unweighted: 4.7629 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4460 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5917 | Regression_eta: 0.0739 | Regression_pt: 0.5467 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6297 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4126 | Classification_binary: 3.5026 | Regression_cos_phi: 1.8229 | Regression_energy: 3.1172 | Regression_eta: 0.9239 | Regression_pt: 4.3581 | Regression_sin_phi: 2.1624 | LR: 4.57e-05 | DataLoad Time: 0.0009s | Model Forward Time: 2.6206s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1620 | max_loss: 0.0037 | max_backward: 0.2478 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 1.1676 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0595 | Memory: rss_MB: 7302.2 | uss_MB: 6481.6 | child_rss_MB: 6997.3 | allocated_MB: 394.0 | reserved_MB: 11392.0 | inactive_split_MB: 90.6 +rank4 [2026-10-27 21:36:11,295] INFO : Step 93000/100000 rank4 | Train Loss: 13.1041 | Train Loss Unweighted: 4.5189 | Train Loss Components: Classification: 0.0119 | Classification_binary: 3.5265 | Regression_cos_phi: 0.0575 | Regression_energy: 0.4342 | Regression_eta: 0.0649 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6346 | physical_nopu: 4.4327 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2486 | Classification_binary: 3.3378 | Regression_cos_phi: 2.1560 | Regression_energy: 1.8965 | Regression_eta: 0.5315 | Regression_pt: 2.3041 | Regression_sin_phi: 1.6951 | LR: 4.43e-05 | DataLoad Time: 0.0022s | Model Forward Time: 2.4219s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1013 | max_loss: 0.1069 | max_backward: 0.2035 | max_optimizer_step: 0.1792 | max_tensorboard_logging: 1.1596 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 5379.1 | uss_MB: 5141.2 | child_rss_MB: 6951.6 | allocated_MB: 393.6 | reserved_MB: 13294.0 | inactive_split_MB: 89.2 +rank4 [2026-10-27 21:39:11,302] INFO : Step 94000/100000 rank4 | Train Loss: 11.5177 | Train Loss Unweighted: 4.2756 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4441 | Regression_cos_phi: 0.0413 | Regression_energy: 0.3503 | Regression_eta: 0.0686 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6308 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3427 | Classification_binary: 3.3765 | Regression_cos_phi: 1.3940 | Regression_energy: 1.6065 | Regression_eta: 0.5347 | Regression_pt: 2.1786 | Regression_sin_phi: 1.2928 | LR: 4.32e-05 | DataLoad Time: 0.0022s | Model Forward Time: 6.3162s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0191 | max_forward: 0.1721 | max_loss: 0.0188 | max_backward: 0.2028 | max_optimizer_step: 0.2895 | max_tensorboard_logging: 4.7123 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 5372.1 | uss_MB: 5157.2 | child_rss_MB: 6925.7 | allocated_MB: 394.1 | reserved_MB: 13294.0 | inactive_split_MB: 89.1 +rank4 [2026-10-27 21:42:20,694] INFO : Step 95000/100000 rank4 | Train Loss: 14.1207 | Train Loss Unweighted: 4.6477 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3920 | Regression_cos_phi: 0.0441 | Regression_energy: 0.5776 | Regression_eta: 0.0602 | Regression_pt: 0.5087 | Regression_sin_phi: 0.0499 | Group Train Loss Unweighted (window mean): physical_pu: 3.6267 | physical_nopu: 4.4321 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3432 | Classification_binary: 3.4223 | Regression_cos_phi: 2.0820 | Regression_energy: 2.8838 | Regression_eta: 0.7439 | Regression_pt: 3.8886 | Regression_sin_phi: 1.4159 | LR: 4.22e-05 | DataLoad Time: 0.0012s | Model Forward Time: 2.5042s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.0941 | max_loss: 0.0036 | max_backward: 0.1734 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 1.2811 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5361.0 | uss_MB: 5157.8 | child_rss_MB: 6928.3 | allocated_MB: 396.0 | reserved_MB: 14294.0 | inactive_split_MB: 91.1 +rank4 [2026-10-27 21:45:16,250] INFO : Step 96000/100000 rank4 | Train Loss: 11.5036 | Train Loss Unweighted: 4.2276 | Train Loss Components: Classification: 0.0097 | Classification_binary: 3.4897 | Regression_cos_phi: 0.0538 | Regression_energy: 0.2982 | Regression_eta: 0.0802 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6278 | physical_nopu: 4.4329 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2069 | Classification_binary: 3.4790 | Regression_cos_phi: 1.6837 | Regression_energy: 1.2385 | Regression_eta: 0.6637 | Regression_pt: 1.5323 | Regression_sin_phi: 1.6845 | LR: 4.14e-05 | DataLoad Time: 0.0024s | Model Forward Time: 2.3867s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0007 | max_forward: 0.0991 | max_loss: 0.0037 | max_backward: 0.1000 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5357.0 | uss_MB: 5158.4 | child_rss_MB: 6899.7 | allocated_MB: 393.0 | reserved_MB: 14294.0 | inactive_split_MB: 95.3 +rank4 [2026-10-27 21:48:34,573] INFO : Step 97000/100000 rank4 | Train Loss: 11.8793 | Train Loss Unweighted: 4.3044 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.4420 | Regression_cos_phi: 0.0454 | Regression_energy: 0.3663 | Regression_eta: 0.0711 | Regression_pt: 0.3156 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.6277 | physical_nopu: 4.4255 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4408 | Classification_binary: 3.2979 | Regression_cos_phi: 1.8252 | Regression_energy: 1.7806 | Regression_eta: 0.7453 | Regression_pt: 2.2691 | Regression_sin_phi: 1.5218 | LR: 4.08e-05 | DataLoad Time: 0.0010s | Model Forward Time: 2.5923s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1192 | max_loss: 0.0048 | max_backward: 0.1852 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 5356.0 | uss_MB: 5157.9 | child_rss_MB: 6903.2 | allocated_MB: 394.7 | reserved_MB: 14294.0 | inactive_split_MB: 98.1 +rank4 [2026-10-27 21:51:32,851] INFO : Step 98000/100000 rank4 | Train Loss: 12.9367 | Train Loss Unweighted: 4.4840 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0441 | Regression_energy: 0.4811 | Regression_eta: 0.0611 | Regression_pt: 0.4378 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3778 | Classification_binary: 3.4440 | Regression_cos_phi: 2.0146 | Regression_energy: 2.4526 | Regression_eta: 0.9607 | Regression_pt: 2.9482 | Regression_sin_phi: 1.5347 | LR: 4.04e-05 | DataLoad Time: 0.0018s | Model Forward Time: 2.8944s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1318 | max_loss: 0.0038 | max_backward: 0.1127 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.6868 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 5362.0 | uss_MB: 5160.3 | child_rss_MB: 6899.7 | allocated_MB: 394.5 | reserved_MB: 14294.0 | inactive_split_MB: 82.0 +rank4 [2026-10-27 21:54:35,739] INFO : Step 99000/100000 rank4 | Train Loss: 11.6408 | Train Loss Unweighted: 4.3204 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.5557 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3117 | Regression_eta: 0.0641 | Regression_pt: 0.2727 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.6292 | physical_nopu: 4.4284 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2464 | Classification_binary: 3.5952 | Regression_cos_phi: 1.9477 | Regression_energy: 1.5360 | Regression_eta: 0.7798 | Regression_pt: 1.9159 | Regression_sin_phi: 1.7069 | LR: 4.01e-05 | DataLoad Time: 0.0084s | Model Forward Time: 3.0807s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1731 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.8331 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 5358.0 | uss_MB: 5160.5 | child_rss_MB: 6886.0 | allocated_MB: 394.3 | reserved_MB: 14294.0 | inactive_split_MB: 113.7 +rank4 [2026-10-27 21:57:56,502] INFO : Step 100000/100000 rank4 | Train Loss: 10.8909 | Train Loss Unweighted: 3.6545 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9133 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2833 | Regression_eta: 0.1041 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6336 | physical_nopu: 4.4311 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3380 | Classification_binary: 2.9984 | Regression_cos_phi: 1.8607 | Regression_energy: 1.3834 | Regression_eta: 1.1138 | Regression_pt: 1.7634 | Regression_sin_phi: 1.7011 | LR: 4.00e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.7390s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0030 | max_forward: 0.2088 | max_loss: 0.0393 | max_backward: 0.3031 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.9276 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 5350.9 | uss_MB: 5163.3 | child_rss_MB: 6834.2 | allocated_MB: 405.7 | reserved_MB: 14294.0 | inactive_split_MB: 165.9 +rank4 [2026-10-27 21:57:57,460] INFO : Running validation on rank4 +rank4 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 21:59:51,634] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 21:59:52,048] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 22:00:18,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 22:00:19,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 22:00:19,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 22:00:20,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 22:00:20,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 22:00:21,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 22:00:21,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 22:00:21,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 22:00:31,957] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 22:00:32,062] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 22:00:32,465] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 22:00:47,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 22:00:48,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 22:00:48,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 22:00:49,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 22:00:49,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 22:00:50,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 22:00:50,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 22:00:51,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 22:01:01,055] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 22:01:01,522] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 22:01:01,928] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 22:01:17,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 22:01:17,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 22:01:18,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 22:01:18,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 22:01:19,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 22:01:19,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 22:01:19,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 22:01:20,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 22:01:30,414] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 22:01:31,011] INFO : Training completed. Total time on device 4: 350.242min diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.5 b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.5 new file mode 100644 index 0000000000000000000000000000000000000000..ce51c8e7437a8320011ad6525afe740e6bcb620b --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.5 @@ -0,0 +1,851 @@ +rank5 [2026-10-27 16:10:10,419] INFO : Initializing model with seed=12345; process seed=12350 +rank5 [2026-10-27 16:10:10,420] INFO : configured dtype=torch.bfloat16 for autocast +rank5 [2026-10-27 16:10:37,169] INFO : Instantiating model +rank5 [2026-10-27 16:10:37,169] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank5 [2026-10-27 16:10:37,169] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank5 [2026-10-27 16:10:37,201] INFO : Input encoding initialization took 0.03s +rank5 [2026-10-27 16:10:37,201] INFO : nn0 parameters: 2130944 +rank5 [2026-10-27 16:10:37,201] INFO : Initializing shared backbone layers, num_convs=3 +rank5 [2026-10-27 16:10:37,201] INFO : Initializing backbone layer 0 +rank5 [2026-10-27 16:10:37,201] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-27 16:10:37,212] INFO : Initializing backbone layer 1 +rank5 [2026-10-27 16:10:37,212] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-27 16:10:37,223] INFO : Initializing backbone layer 2 +rank5 [2026-10-27 16:10:37,223] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-27 16:10:37,233] INFO : Backbone initialization took 0.03s +rank5 [2026-10-27 16:10:37,233] INFO : backbone parameters: 4733952 +rank5 [2026-10-27 16:10:37,233] INFO : Initializing output DNNs +rank5 [2026-10-27 16:10:37,267] INFO : Output DNNs initialization took 0.03s +rank5 [2026-10-27 16:10:37,268] INFO : backbone_mode=BackboneMode.SHARED +rank5 [2026-10-27 16:10:37,268] INFO : jagged_attention enabled=True +rank5 [2026-10-27 16:10:37,268] INFO : flash_attn_varlen enabled=True +rank5 [2026-10-27 16:10:37,268] INFO : task_queries enabled=False +rank5 [2026-10-27 16:10:37,268] INFO : classification_norm parameters: 1024 +rank5 [2026-10-27 16:10:37,268] INFO : regression_norm parameters: 1024 +rank5 [2026-10-27 16:10:37,268] INFO : classification_query parameters: 0 +rank5 [2026-10-27 16:10:37,268] INFO : regression_query parameters: 0 +rank5 [2026-10-27 16:10:37,268] INFO : classification_readout parameters: 0 +rank5 [2026-10-27 16:10:37,268] INFO : regression_readout parameters: 0 +rank5 [2026-10-27 16:10:37,268] INFO : output_mode=elementwise +rank5 [2026-10-27 16:10:37,268] INFO : set_decoder parameters: 0 +rank5 [2026-10-27 16:10:37,268] INFO : nn_binary_particle parameters: 264706 +rank5 [2026-10-27 16:10:37,268] INFO : nn_pid parameters: 268297 +rank5 [2026-10-27 16:10:37,268] INFO : nn_pt parameters: 2113544 +rank5 [2026-10-27 16:10:37,269] INFO : nn_eta parameters: 264706 +rank5 [2026-10-27 16:10:37,269] INFO : nn_sin_phi parameters: 264706 +rank5 [2026-10-27 16:10:37,269] INFO : nn_cos_phi parameters: 264706 +rank5 [2026-10-27 16:10:37,269] INFO : nn_energy parameters: 2113544 +rank5 [2026-10-27 16:10:37,269] INFO : Total MLPF parameters: 12421153 +rank5 [2026-10-27 16:10:37,269] INFO : MLPF __init__ done +rank5 [2026-10-27 16:10:37,270] INFO : Instantiated model +rank5 [2026-10-27 16:10:37,270] INFO : Moving model to device rank=5 +rank5 [2026-10-27 16:10:37,298] INFO : Moved model to device rank=5 +rank5 [2026-10-27 16:10:37,300] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank5 [2026-10-27 16:10:37,300] INFO : Configured model for SyncBatchNorm rank=5 +rank5 [2026-10-27 16:10:37,493] INFO : Configured model for DistributedDataParallel rank=5 +rank5 [2026-10-27 16:10:37,593] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +167 module.nn_energy.nn.7.0.bias ... 0 +168 module.nn_energy.nn.7.2.weight ... 0 +169 module.nn_energy.nn.7.2.bias ... 0 +170 module.nn_energy.nn.7.4.weight ... 0 +171 module.nn_energy.nn.7.4.bias ... 0 + +[172 rows x 3 columns] +rank5 [2026-10-27 16:10:37,593] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=512, out_features=512, bias=True) + ) + (norm0): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=512, out_features=512, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank5 [2026-10-27 16:10:37,594] INFO : Trainable parameters: 12421153 +rank5 [2026-10-27 16:10:37,594] INFO : Non-trainable parameters: 0 +rank5 [2026-10-27 16:10:37,594] INFO : Total parameters: 12421153 +rank5 [2026-10-27 16:10:37,597] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.mha.in_proj_weight 786432 0 + module.backbone.0.mha.in_proj_bias 1536 0 +module.backbone.0.mha.out_proj.weight 262144 0 + module.backbone.0.mha.out_proj.bias 512 0 + module.backbone.0.norm0.weight 512 0 + module.backbone.0.norm0.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm1.bias 512 0 + module.backbone.0.seq.0.weight 262144 0 + module.backbone.0.seq.0.bias 512 0 + module.backbone.0.seq.2.weight 262144 0 + module.backbone.0.seq.2.bias 512 0 + module.backbone.1.mha.in_proj_weight 786432 0 + module.backbone.1.mha.in_proj_bias 1536 0 +module.backbone.1.mha.out_proj.weight 262144 0 + module.backbone.1.mha.out_proj.bias 512 0 + module.backbone.1.norm0.weight 512 0 + module.backbone.1.norm0.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm1.bias 512 0 + module.backbone.1.seq.0.weight 262144 0 + module.backbone.1.seq.0.bias 512 0 + module.backbone.1.seq.2.weight 262144 0 + module.backbone.1.seq.2.bias 512 0 + module.backbone.2.mha.in_proj_weight 786432 0 + module.backbone.2.mha.in_proj_bias 1536 0 +module.backbone.2.mha.out_proj.weight 262144 0 + module.backbone.2.mha.out_proj.bias 512 0 + module.backbone.2.norm0.weight 512 0 + module.backbone.2.norm0.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm1.bias 512 0 + module.backbone.2.seq.0.weight 262144 0 + module.backbone.2.seq.0.bias 512 0 + module.backbone.2.seq.2.weight 262144 0 + module.backbone.2.seq.2.bias 512 0 + module.classification_norm.weight 512 0 + module.classification_norm.bias 512 0 + module.regression_norm.weight 512 0 + module.regression_norm.bias 512 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank5 [2026-10-27 16:10:37,597] INFO : Getting dataloaders +rank5 [2026-10-27 16:10:37,597] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:43,313] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:44,235] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:44,826] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank5 [2026-10-27 16:10:44,827] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:46,036] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:47,297] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:47,787] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank5 [2026-10-27 16:10:47,788] INFO : Creating WeightedMixIterator with 2 data loaders. +rank5 [2026-10-27 16:10:47,788] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:47,946] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:48,094] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:48,245] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank5 [2026-10-27 16:10:48,245] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:48,417] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:48,581] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:48,735] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank5 [2026-10-27 16:10:48,735] INFO : Creating InterleavedIterator with 2 data loaders. +rank5 [2026-10-27 16:11:16,972] INFO : Got dataloaders +rank5 [2026-10-27 16:11:16,973] INFO : loader split=train rank=5 len=232058 +rank5 [2026-10-27 16:11:16,973] INFO : loader split=valid rank=5 len=48 +rank5 [2026-10-27 16:11:16,973] INFO : Starting training from step 1 to 100000 on rank 5 of 8 +rank5 [2026-10-27 16:11:16,973] INFO : Creating train_iterator +rank5 [2026-10-27 16:11:16,973] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank5 [2026-10-27 16:11:30,947] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank5 [2026-10-27 16:11:46,090] INFO : Created train_iterator +rank5 [2026-10-27 16:12:23,643] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.123548031449318, 'Classification': 0.2298217851272784, 'Regression_pt': 0.7194055180624127, 'Regression_eta': 0.4619348940253258, 'Regression_sin_phi': 0.15005176125559955, 'Regression_cos_phi': 0.14165307599119842, 'Regression_energy': 1.0569261094555258}, weights={'Classification_binary': 1.0, 'Classification': 22.293569564819336, 'Regression_pt': 7.12191915512085, 'Regression_eta': 11.091493606567383, 'Regression_sin_phi': 34.14520263671875, 'Regression_cos_phi': 36.16969299316406, 'Regression_energy': 4.847593307495117} +rank5 [2026-10-27 16:14:48,546] INFO : Step 1000/100000 rank5 | Train Loss: 15.9738 | Train Loss Unweighted: 5.5021 | Train Loss Components: Classification: 0.0191 | Classification_binary: 4.1436 | Regression_cos_phi: 0.0516 | Regression_energy: 0.6035 | Regression_eta: 0.0819 | Regression_pt: 0.5500 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 5.2846 | physical_nopu: 6.1663 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3783 | Classification_binary: 4.2352 | Regression_cos_phi: 1.5479 | Regression_energy: 2.6197 | Regression_eta: 0.9809 | Regression_pt: 3.6628 | Regression_sin_phi: 1.5565 | LR: 4.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9580s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0013 | max_forward: 0.0194 | max_loss: 0.0039 | max_backward: 0.0308 | max_optimizer_step: 0.1391 | max_tensorboard_logging: 1.0816 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0030 | Memory: rss_MB: 4094.8 | uss_MB: 2885.6 | child_rss_MB: 10815.8 | allocated_MB: 395.5 | reserved_MB: 11198.0 | inactive_split_MB: 104.9 +rank5 [2026-10-27 16:17:31,218] INFO : Step 2000/100000 rank5 | Train Loss: 17.2752 | Train Loss Unweighted: 5.6831 | Train Loss Components: Classification: 0.0157 | Classification_binary: 4.0887 | Regression_cos_phi: 0.0506 | Regression_energy: 0.7236 | Regression_eta: 0.0696 | Regression_pt: 0.6798 | Regression_sin_phi: 0.0552 | Group Train Loss Unweighted (window mean): physical_pu: 4.4021 | physical_nopu: 5.4797 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3113 | Classification_binary: 4.1066 | Regression_cos_phi: 1.8098 | Regression_energy: 2.8171 | Regression_eta: 0.7347 | Regression_pt: 3.8735 | Regression_sin_phi: 2.3764 | LR: 4.00e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.7798s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0250 | max_loss: 0.0036 | max_backward: 0.0657 | max_optimizer_step: 0.1397 | max_tensorboard_logging: 0.8949 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4097.7 | uss_MB: 2895.6 | child_rss_MB: 10834.7 | allocated_MB: 394.2 | reserved_MB: 11198.0 | inactive_split_MB: 90.1 +rank5 [2026-10-27 16:20:19,985] INFO : Step 3000/100000 rank5 | Train Loss: 14.0295 | Train Loss Unweighted: 5.1258 | Train Loss Components: Classification: 0.0148 | Classification_binary: 4.0405 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4749 | Regression_eta: 0.0746 | Regression_pt: 0.4205 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 4.1913 | physical_nopu: 5.3261 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3678 | Classification_binary: 3.8948 | Regression_cos_phi: 1.6907 | Regression_energy: 2.2035 | Regression_eta: 0.6913 | Regression_pt: 2.9260 | Regression_sin_phi: 1.7220 | LR: 3.99e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.4926s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1067 | max_loss: 0.0036 | max_backward: 0.0365 | max_optimizer_step: 0.1415 | max_tensorboard_logging: 1.1098 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0093 | Memory: rss_MB: 4102.6 | uss_MB: 2907.4 | child_rss_MB: 10837.4 | allocated_MB: 395.4 | reserved_MB: 11784.0 | inactive_split_MB: 88.5 +rank5 [2026-10-27 16:23:08,050] INFO : Step 4000/100000 rank5 | Train Loss: 11.7244 | Train Loss Unweighted: 3.9925 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.1718 | Regression_cos_phi: 0.0487 | Regression_energy: 0.3212 | Regression_eta: 0.1125 | Regression_pt: 0.2729 | Regression_sin_phi: 0.0492 | Group Train Loss Unweighted (window mean): physical_pu: 4.0838 | physical_nopu: 5.2370 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3848 | Classification_binary: 3.1886 | Regression_cos_phi: 1.7421 | Regression_energy: 1.6431 | Regression_eta: 1.3105 | Regression_pt: 2.0615 | Regression_sin_phi: 1.7376 | LR: 3.99e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.2686s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0037 | max_forward: 0.0726 | max_loss: 0.0066 | max_backward: 0.1670 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4108.6 | uss_MB: 2925.8 | child_rss_MB: 10838.2 | allocated_MB: 406.4 | reserved_MB: 11904.0 | inactive_split_MB: 179.6 +rank5 [2026-10-27 16:25:55,983] INFO : Step 5000/100000 rank5 | Train Loss: 13.1313 | Train Loss Unweighted: 4.9481 | Train Loss Components: Classification: 0.0138 | Classification_binary: 4.0144 | Regression_cos_phi: 0.0518 | Regression_energy: 0.3902 | Regression_eta: 0.0928 | Regression_pt: 0.3380 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 4.0057 | physical_nopu: 5.1582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3358 | Classification_binary: 3.9327 | Regression_cos_phi: 1.9189 | Regression_energy: 1.6691 | Regression_eta: 1.0888 | Regression_pt: 2.1343 | Regression_sin_phi: 1.5988 | LR: 3.98e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.3570s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0196 | max_loss: 0.0037 | max_backward: 0.0324 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 1.3896 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0068 | Memory: rss_MB: 4091.1 | uss_MB: 2927.1 | child_rss_MB: 10673.2 | allocated_MB: 393.9 | reserved_MB: 11904.0 | inactive_split_MB: 90.0 +rank5 [2026-10-27 16:28:58,362] INFO : Step 6000/100000 rank5 | Train Loss: 14.5226 | Train Loss Unweighted: 5.0639 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.8601 | Regression_cos_phi: 0.0525 | Regression_energy: 0.5359 | Regression_eta: 0.0694 | Regression_pt: 0.4825 | Regression_sin_phi: 0.0459 | Group Train Loss Unweighted (window mean): physical_pu: 3.9666 | physical_nopu: 5.0829 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3410 | Classification_binary: 3.8948 | Regression_cos_phi: 1.9361 | Regression_energy: 2.3135 | Regression_eta: 0.6336 | Regression_pt: 3.0986 | Regression_sin_phi: 1.4710 | LR: 3.97e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.5987s | Phase Times: max_pre_step_barrier: 0.0280 | max_batch_to_device: 0.0008 | max_forward: 0.0311 | max_loss: 0.0037 | max_backward: 0.0629 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 2.6604 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0119 | Memory: rss_MB: 4078.1 | uss_MB: 2934.4 | child_rss_MB: 10495.3 | allocated_MB: 394.6 | reserved_MB: 12524.0 | inactive_split_MB: 109.0 +rank5 [2026-10-27 16:31:51,870] INFO : Step 7000/100000 rank5 | Train Loss: 13.9772 | Train Loss Unweighted: 5.0398 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.9636 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4671 | Regression_eta: 0.0797 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0537 | Group Train Loss Unweighted (window mean): physical_pu: 3.9390 | physical_nopu: 5.0316 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1821 | Classification_binary: 3.8907 | Regression_cos_phi: 2.1140 | Regression_energy: 1.5823 | Regression_eta: 0.9282 | Regression_pt: 1.7527 | Regression_sin_phi: 1.8380 | LR: 3.96e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.1743s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0243 | max_loss: 0.0036 | max_backward: 0.0295 | max_optimizer_step: 0.2429 | max_tensorboard_logging: 2.1406 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 4070.6 | uss_MB: 2937.8 | child_rss_MB: 10370.1 | allocated_MB: 393.9 | reserved_MB: 12524.0 | inactive_split_MB: 107.5 +rank5 [2026-10-27 16:35:03,086] INFO : Step 8000/100000 rank5 | Train Loss: 13.6161 | Train Loss Unweighted: 4.9650 | Train Loss Components: Classification: 0.0187 | Classification_binary: 3.9275 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4519 | Regression_eta: 0.0615 | Regression_pt: 0.4056 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.9031 | physical_nopu: 4.9878 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4377 | Classification_binary: 3.8218 | Regression_cos_phi: 1.8222 | Regression_energy: 1.7996 | Regression_eta: 0.6014 | Regression_pt: 2.3219 | Regression_sin_phi: 1.1943 | LR: 3.94e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.9575s | Phase Times: max_pre_step_barrier: 0.0692 | max_batch_to_device: 0.0893 | max_forward: 0.1169 | max_loss: 0.0037 | max_backward: 0.0488 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 1.9012 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4069.6 | uss_MB: 2945.4 | child_rss_MB: 10376.1 | allocated_MB: 393.8 | reserved_MB: 12524.0 | inactive_split_MB: 105.1 +rank5 [2026-10-27 16:38:07,477] INFO : Step 9000/100000 rank5 | Train Loss: 15.1606 | Train Loss Unweighted: 5.0855 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.7493 | Regression_cos_phi: 0.0474 | Regression_energy: 0.5909 | Regression_eta: 0.0749 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.8927 | physical_nopu: 4.9579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4218 | Classification_binary: 3.7523 | Regression_cos_phi: 1.7921 | Regression_energy: 3.4920 | Regression_eta: 0.6068 | Regression_pt: 4.7680 | Regression_sin_phi: 1.7738 | LR: 3.93e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.8501s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0007 | max_forward: 0.0386 | max_loss: 0.0041 | max_backward: 0.0582 | max_optimizer_step: 0.1496 | max_tensorboard_logging: 1.9225 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 4056.6 | uss_MB: 2948.9 | child_rss_MB: 10378.8 | allocated_MB: 395.8 | reserved_MB: 12524.0 | inactive_split_MB: 89.5 +rank5 [2026-10-27 16:41:18,312] INFO : Step 10000/100000 rank5 | Train Loss: 12.2525 | Train Loss Unweighted: 4.6725 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8667 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3374 | Regression_eta: 0.0750 | Regression_pt: 0.2772 | Regression_sin_phi: 0.0525 | Group Train Loss Unweighted (window mean): physical_pu: 3.8762 | physical_nopu: 4.9262 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2428 | Classification_binary: 3.8077 | Regression_cos_phi: 1.5094 | Regression_energy: 1.9310 | Regression_eta: 0.6878 | Regression_pt: 2.3714 | Regression_sin_phi: 1.8508 | LR: 3.91e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.1910s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0277 | max_loss: 0.0035 | max_backward: 0.0304 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 2.2107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4024.3 | uss_MB: 2953.8 | child_rss_MB: 10382.4 | allocated_MB: 393.6 | reserved_MB: 12524.0 | inactive_split_MB: 89.8 +rank5 [2026-10-27 16:41:19,252] INFO : Running validation on rank5 +rank5 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 16:43:24,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:43:25,081] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 16:43:52,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 16:43:53,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 16:43:53,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 16:43:54,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 16:43:54,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 16:43:55,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 16:43:55,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 16:43:56,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 16:44:06,053] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-27 16:44:06,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:44:06,938] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 16:44:22,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 16:44:22,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 16:44:23,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 16:44:23,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 16:44:24,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 16:44:24,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 16:44:25,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 16:44:25,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 16:44:35,643] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 16:44:35,783] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:44:36,273] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 16:44:50,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 16:44:51,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 16:44:51,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 16:44:52,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 16:44:52,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 16:44:53,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 16:44:53,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 16:44:54,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 16:45:04,141] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 16:48:51,455] INFO : Step 11000/100000 rank5 | Train Loss: 13.5331 | Train Loss Unweighted: 4.8490 | Train Loss Components: Classification: 0.0178 | Classification_binary: 3.8332 | Regression_cos_phi: 0.0514 | Regression_energy: 0.4326 | Regression_eta: 0.0715 | Regression_pt: 0.3906 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.8629 | physical_nopu: 4.8860 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4138 | Classification_binary: 3.9033 | Regression_cos_phi: 1.6749 | Regression_energy: 2.1958 | Regression_eta: 0.9524 | Regression_pt: 2.7210 | Regression_sin_phi: 1.8010 | LR: 3.89e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.1304s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0010 | max_forward: 0.0938 | max_loss: 0.0036 | max_backward: 0.1077 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.9405 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6748.2 | uss_MB: 5693.6 | child_rss_MB: 10378.2 | allocated_MB: 394.1 | reserved_MB: 11016.0 | inactive_split_MB: 102.9 +rank5 [2026-10-27 16:51:46,972] INFO : Step 12000/100000 rank5 | Train Loss: 16.0880 | Train Loss Unweighted: 5.2955 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.7102 | Regression_cos_phi: 0.0409 | Regression_energy: 0.7300 | Regression_eta: 0.0621 | Regression_pt: 0.6963 | Regression_sin_phi: 0.0392 | Group Train Loss Unweighted (window mean): physical_pu: 3.8522 | physical_nopu: 4.8676 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3773 | Classification_binary: 3.5911 | Regression_cos_phi: 1.0778 | Regression_energy: 3.3292 | Regression_eta: 0.5011 | Regression_pt: 4.7382 | Regression_sin_phi: 1.1529 | LR: 3.87e-04 | DataLoad Time: 0.0011s | Model Forward Time: 2.9542s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1050 | max_forward: 0.2073 | max_loss: 0.0038 | max_backward: 0.1040 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 1.6163 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6734.2 | uss_MB: 5695.7 | child_rss_MB: 10317.1 | allocated_MB: 394.0 | reserved_MB: 11384.0 | inactive_split_MB: 88.0 +rank5 [2026-10-27 16:54:47,130] INFO : Step 13000/100000 rank5 | Train Loss: 12.2526 | Train Loss Unweighted: 4.6994 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.8775 | Regression_cos_phi: 0.0480 | Regression_energy: 0.3426 | Regression_eta: 0.0688 | Regression_pt: 0.2970 | Regression_sin_phi: 0.0539 | Group Train Loss Unweighted (window mean): physical_pu: 3.8404 | physical_nopu: 4.8415 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2356 | Classification_binary: 3.9630 | Regression_cos_phi: 1.7846 | Regression_energy: 1.9784 | Regression_eta: 0.7334 | Regression_pt: 2.5771 | Regression_sin_phi: 1.7197 | LR: 3.85e-04 | DataLoad Time: 0.0051s | Model Forward Time: 3.3098s | Phase Times: max_pre_step_barrier: 0.0143 | max_batch_to_device: 0.0010 | max_forward: 0.0740 | max_loss: 0.0036 | max_backward: 0.0639 | max_optimizer_step: 0.2642 | max_tensorboard_logging: 2.1016 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 5345.9 | uss_MB: 4318.0 | child_rss_MB: 10308.5 | allocated_MB: 394.7 | reserved_MB: 12168.0 | inactive_split_MB: 111.5 +rank5 [2026-10-27 16:57:46,633] INFO : Step 14000/100000 rank5 | Train Loss: 11.1328 | Train Loss Unweighted: 3.8032 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.0595 | Regression_cos_phi: 0.0494 | Regression_energy: 0.2823 | Regression_eta: 0.1077 | Regression_pt: 0.2396 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.8281 | physical_nopu: 4.8178 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3607 | Classification_binary: 3.1139 | Regression_cos_phi: 1.7351 | Regression_energy: 1.3768 | Regression_eta: 1.1104 | Regression_pt: 1.7347 | Regression_sin_phi: 1.7479 | LR: 3.83e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.7655s | Phase Times: max_pre_step_barrier: 0.1064 | max_batch_to_device: 0.0033 | max_forward: 0.1130 | max_loss: 0.0036 | max_backward: 0.2788 | max_optimizer_step: 0.1663 | max_tensorboard_logging: 2.1758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1099 | Memory: rss_MB: 5335.9 | uss_MB: 4329.8 | child_rss_MB: 10259.4 | allocated_MB: 405.3 | reserved_MB: 12168.0 | inactive_split_MB: 126.3 +rank5 [2026-10-27 17:00:50,805] INFO : Step 15000/100000 rank5 | Train Loss: 12.4223 | Train Loss Unweighted: 4.6301 | Train Loss Components: Classification: 0.0132 | Classification_binary: 3.7975 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3484 | Regression_eta: 0.0765 | Regression_pt: 0.2883 | Regression_sin_phi: 0.0502 | Group Train Loss Unweighted (window mean): physical_pu: 3.8205 | physical_nopu: 4.8025 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2650 | Classification_binary: 4.0145 | Regression_cos_phi: 2.3860 | Regression_energy: 1.5564 | Regression_eta: 0.7120 | Regression_pt: 2.0555 | Regression_sin_phi: 1.7878 | LR: 3.80e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.2229s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0799 | max_loss: 0.0851 | max_backward: 0.1383 | max_optimizer_step: 0.1621 | max_tensorboard_logging: 2.1079 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5320.9 | uss_MB: 4324.4 | child_rss_MB: 10137.7 | allocated_MB: 394.1 | reserved_MB: 12168.0 | inactive_split_MB: 89.6 +rank5 [2026-10-27 17:04:12,219] INFO : Step 16000/100000 rank5 | Train Loss: 13.1559 | Train Loss Unweighted: 4.6950 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.6917 | Regression_cos_phi: 0.0473 | Regression_energy: 0.4330 | Regression_eta: 0.0771 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.8147 | physical_nopu: 4.7795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3187 | Classification_binary: 3.6346 | Regression_cos_phi: 1.4549 | Regression_energy: 1.8892 | Regression_eta: 0.9519 | Regression_pt: 2.3269 | Regression_sin_phi: 1.9658 | LR: 3.78e-04 | DataLoad Time: 0.0029s | Model Forward Time: 3.2748s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.0598 | max_loss: 0.0036 | max_backward: 0.1881 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 2.1012 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5304.0 | uss_MB: 4326.3 | child_rss_MB: 10027.9 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 88.9 +rank5 [2026-10-27 17:07:14,836] INFO : Step 17000/100000 rank5 | Train Loss: 13.6189 | Train Loss Unweighted: 4.8180 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7348 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4756 | Regression_eta: 0.0731 | Regression_pt: 0.4225 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.8095 | physical_nopu: 4.7681 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2936 | Classification_binary: 3.7969 | Regression_cos_phi: 1.8146 | Regression_energy: 2.2929 | Regression_eta: 1.2317 | Regression_pt: 2.9228 | Regression_sin_phi: 1.4560 | LR: 3.75e-04 | DataLoad Time: 0.0022s | Model Forward Time: 4.0402s | Phase Times: max_pre_step_barrier: 0.6520 | max_batch_to_device: 0.0829 | max_forward: 0.1388 | max_loss: 0.0036 | max_backward: 0.0636 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 2.2240 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5273.0 | uss_MB: 4329.2 | child_rss_MB: 9776.7 | allocated_MB: 394.3 | reserved_MB: 12168.0 | inactive_split_MB: 88.4 +rank5 [2026-10-27 17:10:19,258] INFO : Step 18000/100000 rank5 | Train Loss: 12.6032 | Train Loss Unweighted: 4.6266 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7426 | Regression_cos_phi: 0.0507 | Regression_energy: 0.3730 | Regression_eta: 0.0736 | Regression_pt: 0.3195 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7954 | physical_nopu: 4.7518 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3292 | Classification_binary: 3.7702 | Regression_cos_phi: 1.7709 | Regression_energy: 1.4262 | Regression_eta: 0.7145 | Regression_pt: 1.7783 | Regression_sin_phi: 1.4542 | LR: 3.72e-04 | DataLoad Time: 0.0019s | Model Forward Time: 4.2069s | Phase Times: max_pre_step_barrier: 0.0217 | max_batch_to_device: 0.0009 | max_forward: 0.1212 | max_loss: 0.0040 | max_backward: 0.1670 | max_optimizer_step: 0.1610 | max_tensorboard_logging: 3.0399 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5245.9 | uss_MB: 4331.6 | child_rss_MB: 9623.0 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 87.1 +rank5 [2026-10-27 17:13:21,531] INFO : Step 19000/100000 rank5 | Train Loss: 14.6123 | Train Loss Unweighted: 5.0036 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.6735 | Regression_cos_phi: 0.0398 | Regression_energy: 0.6019 | Regression_eta: 0.0612 | Regression_pt: 0.5669 | Regression_sin_phi: 0.0441 | Group Train Loss Unweighted (window mean): physical_pu: 3.7887 | physical_nopu: 4.7426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4014 | Classification_binary: 3.5951 | Regression_cos_phi: 1.4116 | Regression_energy: 4.3120 | Regression_eta: 0.6278 | Regression_pt: 6.3259 | Regression_sin_phi: 1.1954 | LR: 3.69e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.4967s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1593 | max_loss: 0.0094 | max_backward: 0.0749 | max_optimizer_step: 0.1658 | max_tensorboard_logging: 2.3024 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5218.9 | uss_MB: 4333.7 | child_rss_MB: 9466.4 | allocated_MB: 394.5 | reserved_MB: 12168.0 | inactive_split_MB: 103.1 +rank5 [2026-10-27 17:16:34,024] INFO : Step 20000/100000 rank5 | Train Loss: 13.1046 | Train Loss Unweighted: 4.7571 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.7799 | Regression_cos_phi: 0.0517 | Regression_energy: 0.4251 | Regression_eta: 0.0654 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0511 | Group Train Loss Unweighted (window mean): physical_pu: 3.7835 | physical_nopu: 4.7206 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2932 | Classification_binary: 3.6421 | Regression_cos_phi: 1.9983 | Regression_energy: 2.1524 | Regression_eta: 0.9182 | Regression_pt: 2.8099 | Regression_sin_phi: 1.8092 | LR: 3.66e-04 | DataLoad Time: 0.0032s | Model Forward Time: 3.2222s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0008 | max_forward: 0.1344 | max_loss: 0.0037 | max_backward: 0.0763 | max_optimizer_step: 0.1649 | max_tensorboard_logging: 1.9693 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5216.3 | uss_MB: 4336.2 | child_rss_MB: 9462.6 | allocated_MB: 392.8 | reserved_MB: 12168.0 | inactive_split_MB: 89.4 +rank5 [2026-10-27 17:16:34,968] INFO : Running validation on rank5 +rank5 [2026-10-27 17:16:34,968] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 17:16:34,968] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 17:18:27,486] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 17:18:27,879] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 17:18:53,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 17:18:54,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 17:18:54,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 17:18:55,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 17:18:55,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 17:18:56,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 17:18:56,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 17:18:57,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 17:19:07,047] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 17:19:07,312] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 17:19:07,735] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 17:19:22,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 17:19:22,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 17:19:23,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 17:19:23,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 17:19:24,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 17:19:24,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 17:19:25,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 17:19:25,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 17:19:35,826] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 17:19:36,428] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 17:19:36,812] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 17:19:52,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 17:19:52,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 17:19:52,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 17:19:53,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 17:19:53,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 17:19:54,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 17:19:54,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 17:19:55,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 17:20:05,123] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-27 17:23:25,419] INFO : Step 21000/100000 rank5 | Train Loss: 12.4320 | Train Loss Unweighted: 4.5235 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.6041 | Regression_cos_phi: 0.0444 | Regression_energy: 0.3902 | Regression_eta: 0.0812 | Regression_pt: 0.3383 | Regression_sin_phi: 0.0478 | Group Train Loss Unweighted (window mean): physical_pu: 3.7732 | physical_nopu: 4.7052 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4307 | Classification_binary: 3.5234 | Regression_cos_phi: 1.6784 | Regression_energy: 1.9158 | Regression_eta: 0.6991 | Regression_pt: 2.4005 | Regression_sin_phi: 1.4923 | LR: 3.62e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.7289s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.2105 | max_loss: 0.0056 | max_backward: 0.1347 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 2.3606 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5326.2 | uss_MB: 4454.3 | child_rss_MB: 9450.4 | allocated_MB: 394.6 | reserved_MB: 11062.0 | inactive_split_MB: 89.8 +rank5 [2026-10-27 17:26:14,701] INFO : Step 22000/100000 rank5 | Train Loss: 14.5232 | Train Loss Unweighted: 4.8893 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.6034 | Regression_cos_phi: 0.0476 | Regression_energy: 0.5714 | Regression_eta: 0.0738 | Regression_pt: 0.5351 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.7709 | physical_nopu: 4.6938 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2567 | Classification_binary: 3.6988 | Regression_cos_phi: 1.6772 | Regression_energy: 1.6138 | Regression_eta: 0.6506 | Regression_pt: 2.0236 | Regression_sin_phi: 1.3028 | LR: 3.59e-04 | DataLoad Time: 0.0028s | Model Forward Time: 2.6768s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0010 | max_forward: 0.0774 | max_loss: 0.0142 | max_backward: 0.0921 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.4442 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1027 | Memory: rss_MB: 5327.2 | uss_MB: 4458.5 | child_rss_MB: 9447.9 | allocated_MB: 394.2 | reserved_MB: 11488.0 | inactive_split_MB: 86.8 +rank5 [2026-10-27 17:29:50,014] INFO : Step 23000/100000 rank5 | Train Loss: 12.2140 | Train Loss Unweighted: 4.5907 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.7888 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3388 | Regression_eta: 0.0762 | Regression_pt: 0.2687 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.7690 | physical_nopu: 4.6810 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2263 | Classification_binary: 3.7939 | Regression_cos_phi: 1.8145 | Regression_energy: 1.8974 | Regression_eta: 0.9574 | Regression_pt: 2.1279 | Regression_sin_phi: 2.0029 | LR: 3.55e-04 | DataLoad Time: 0.0097s | Model Forward Time: 4.1190s | Phase Times: max_pre_step_barrier: 0.7590 | max_batch_to_device: 0.0007 | max_forward: 0.0763 | max_loss: 0.0189 | max_backward: 0.1117 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 2.0913 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5325.2 | uss_MB: 4459.8 | child_rss_MB: 9460.6 | allocated_MB: 394.0 | reserved_MB: 12220.0 | inactive_split_MB: 178.1 +rank5 [2026-10-27 17:33:22,189] INFO : Step 24000/100000 rank5 | Train Loss: 15.4786 | Train Loss Unweighted: 4.9908 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.5301 | Regression_cos_phi: 0.0467 | Regression_energy: 0.6682 | Regression_eta: 0.0624 | Regression_pt: 0.6223 | Regression_sin_phi: 0.0450 | Group Train Loss Unweighted (window mean): physical_pu: 3.7564 | physical_nopu: 4.6761 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4227 | Classification_binary: 3.2246 | Regression_cos_phi: 0.8740 | Regression_energy: 4.7831 | Regression_eta: 0.4653 | Regression_pt: 6.9395 | Regression_sin_phi: 0.8042 | LR: 3.51e-04 | DataLoad Time: 0.0012s | Model Forward Time: 2.9323s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0719 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1672 | max_tensorboard_logging: 1.8279 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5318.8 | uss_MB: 4462.5 | child_rss_MB: 9451.5 | allocated_MB: 393.8 | reserved_MB: 12220.0 | inactive_split_MB: 105.1 +rank5 [2026-10-27 17:36:26,064] INFO : Step 25000/100000 rank5 | Train Loss: 12.1246 | Train Loss Unweighted: 4.4783 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.7007 | Regression_cos_phi: 0.0551 | Regression_energy: 0.3196 | Regression_eta: 0.0694 | Regression_pt: 0.2642 | Regression_sin_phi: 0.0578 | Group Train Loss Unweighted (window mean): physical_pu: 3.7531 | physical_nopu: 4.6582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2303 | Classification_binary: 3.7856 | Regression_cos_phi: 2.2311 | Regression_energy: 1.5004 | Regression_eta: 0.5918 | Regression_pt: 2.1317 | Regression_sin_phi: 2.0971 | LR: 3.47e-04 | DataLoad Time: 0.0015s | Model Forward Time: 3.0307s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.1917 | max_loss: 0.0039 | max_backward: 0.1029 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 5304.8 | uss_MB: 4464.1 | child_rss_MB: 9445.8 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 86.4 +rank5 [2026-10-27 17:39:30,100] INFO : Step 26000/100000 rank5 | Train Loss: 12.6920 | Train Loss Unweighted: 4.5479 | Train Loss Components: Classification: 0.0188 | Classification_binary: 3.6003 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4034 | Regression_eta: 0.0734 | Regression_pt: 0.3564 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.7511 | physical_nopu: 4.6505 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3942 | Classification_binary: 3.7477 | Regression_cos_phi: 1.8310 | Regression_energy: 2.1230 | Regression_eta: 0.9869 | Regression_pt: 2.5387 | Regression_sin_phi: 1.6933 | LR: 3.43e-04 | DataLoad Time: 0.0008s | Model Forward Time: 3.0952s | Phase Times: max_pre_step_barrier: 0.0479 | max_batch_to_device: 0.0008 | max_forward: 0.1278 | max_loss: 0.0037 | max_backward: 0.0877 | max_optimizer_step: 0.1868 | max_tensorboard_logging: 1.8908 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5292.2 | uss_MB: 4467.2 | child_rss_MB: 9407.7 | allocated_MB: 395.0 | reserved_MB: 12220.0 | inactive_split_MB: 115.0 +rank5 [2026-10-27 17:42:32,922] INFO : Step 27000/100000 rank5 | Train Loss: 16.2758 | Train Loss Unweighted: 5.1421 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5608 | Regression_cos_phi: 0.0448 | Regression_energy: 0.7180 | Regression_eta: 0.0780 | Regression_pt: 0.6791 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.7443 | physical_nopu: 4.6426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2855 | Classification_binary: 3.6376 | Regression_cos_phi: 1.6896 | Regression_energy: 3.7666 | Regression_eta: 1.0043 | Regression_pt: 5.2962 | Regression_sin_phi: 1.6059 | LR: 3.39e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.0076s | Phase Times: max_pre_step_barrier: 0.0385 | max_batch_to_device: 0.0009 | max_forward: 0.0708 | max_loss: 0.0038 | max_backward: 0.0914 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.8878 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5283.2 | uss_MB: 4469.8 | child_rss_MB: 9400.6 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 87.9 +rank5 [2026-10-27 17:45:38,005] INFO : Step 28000/100000 rank5 | Train Loss: 11.9115 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6175 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3339 | Regression_eta: 0.0761 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.7393 | physical_nopu: 4.6288 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2389 | Classification_binary: 3.6553 | Regression_cos_phi: 1.5953 | Regression_energy: 1.7637 | Regression_eta: 0.8211 | Regression_pt: 2.1921 | Regression_sin_phi: 1.6184 | LR: 3.35e-04 | DataLoad Time: 0.0052s | Model Forward Time: 4.7990s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0008 | max_forward: 0.0886 | max_loss: 0.0038 | max_backward: 0.1193 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 3.6285 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 5274.2 | uss_MB: 4146.2 | child_rss_MB: 9355.1 | allocated_MB: 394.0 | reserved_MB: 12414.0 | inactive_split_MB: 106.5 +rank5 [2026-10-27 17:48:54,715] INFO : Step 29000/100000 rank5 | Train Loss: 13.9221 | Train Loss Unweighted: 4.6615 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.5075 | Regression_cos_phi: 0.0491 | Regression_energy: 0.5008 | Regression_eta: 0.0786 | Regression_pt: 0.4589 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7405 | physical_nopu: 4.6233 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3951 | Classification_binary: 3.6149 | Regression_cos_phi: 2.1465 | Regression_energy: 2.2645 | Regression_eta: 0.9678 | Regression_pt: 2.8572 | Regression_sin_phi: 2.1056 | LR: 3.30e-04 | DataLoad Time: 0.0662s | Model Forward Time: 3.3169s | Phase Times: max_pre_step_barrier: 0.1677 | max_batch_to_device: 0.0009 | max_forward: 0.0754 | max_loss: 0.0038 | max_backward: 0.0825 | max_optimizer_step: 0.1673 | max_tensorboard_logging: 2.1272 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5252.2 | uss_MB: 4453.2 | child_rss_MB: 9260.7 | allocated_MB: 394.9 | reserved_MB: 12414.0 | inactive_split_MB: 107.1 +rank5 [2026-10-27 17:51:57,834] INFO : Step 30000/100000 rank5 | Train Loss: 12.4988 | Train Loss Unweighted: 4.5639 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.6431 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3981 | Regression_eta: 0.0722 | Regression_pt: 0.3405 | Regression_sin_phi: 0.0498 | Group Train Loss Unweighted (window mean): physical_pu: 3.7342 | physical_nopu: 4.6125 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2689 | Classification_binary: 3.5361 | Regression_cos_phi: 1.5534 | Regression_energy: 1.5014 | Regression_eta: 0.7527 | Regression_pt: 1.9841 | Regression_sin_phi: 1.6956 | LR: 3.26e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.4587s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0762 | max_loss: 0.0136 | max_backward: 0.0854 | max_optimizer_step: 0.1722 | max_tensorboard_logging: 1.3349 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5242.2 | uss_MB: 4475.0 | child_rss_MB: 9193.8 | allocated_MB: 393.7 | reserved_MB: 12480.0 | inactive_split_MB: 89.7 +rank5 [2026-10-27 17:51:58,780] INFO : Running validation on rank5 +rank5 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 17:51:58,781] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 17:51:58,781] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 17:53:53,312] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 17:53:53,723] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 17:54:18,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 17:54:18,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 17:54:19,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 17:54:19,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 17:54:20,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 17:54:20,903] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 17:54:21,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 17:54:21,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 17:54:31,831] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 17:54:31,840] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 17:54:32,213] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 17:54:47,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 17:54:47,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 17:54:48,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 17:54:48,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 17:54:49,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 17:54:49,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 17:54:50,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 17:54:50,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 17:55:00,598] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 17:55:00,637] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 17:55:01,038] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 17:55:16,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 17:55:16,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 17:55:17,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 17:55:17,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 17:55:18,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 17:55:18,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 17:55:19,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 17:55:19,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 17:55:29,546] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 17:58:47,285] INFO : Step 31000/100000 rank5 | Train Loss: 12.0027 | Train Loss Unweighted: 4.4503 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.5779 | Regression_cos_phi: 0.0460 | Regression_energy: 0.3731 | Regression_eta: 0.0704 | Regression_pt: 0.3230 | Regression_sin_phi: 0.0451 | Group Train Loss Unweighted (window mean): physical_pu: 3.7262 | physical_nopu: 4.6083 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3379 | Classification_binary: 3.6453 | Regression_cos_phi: 1.5882 | Regression_energy: 1.5147 | Regression_eta: 0.7251 | Regression_pt: 2.0418 | Regression_sin_phi: 1.7786 | LR: 3.21e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.5857s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.1458 | max_loss: 0.0037 | max_backward: 0.1256 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 2.3030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7018.1 | uss_MB: 6249.8 | child_rss_MB: 9197.9 | allocated_MB: 394.8 | reserved_MB: 11168.0 | inactive_split_MB: 96.2 +rank5 [2026-10-27 18:01:54,932] INFO : Step 32000/100000 rank5 | Train Loss: 13.5217 | Train Loss Unweighted: 4.6918 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.5357 | Regression_cos_phi: 0.0443 | Regression_energy: 0.5169 | Regression_eta: 0.0728 | Regression_pt: 0.4649 | Regression_sin_phi: 0.0409 | Group Train Loss Unweighted (window mean): physical_pu: 3.7264 | physical_nopu: 4.6006 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2745 | Classification_binary: 3.6013 | Regression_cos_phi: 1.4120 | Regression_energy: 2.0534 | Regression_eta: 0.7657 | Regression_pt: 2.4409 | Regression_sin_phi: 1.3939 | LR: 3.16e-04 | DataLoad Time: 0.0082s | Model Forward Time: 2.3901s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.0795 | max_loss: 0.0917 | max_backward: 0.1851 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.1890 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5310.0 | uss_MB: 4562.1 | child_rss_MB: 9157.4 | allocated_MB: 394.9 | reserved_MB: 11240.0 | inactive_split_MB: 92.3 +rank5 [2026-10-27 18:04:57,678] INFO : Step 33000/100000 rank5 | Train Loss: 11.0126 | Train Loss Unweighted: 3.7465 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9928 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2905 | Regression_eta: 0.1058 | Regression_pt: 0.2472 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.7276 | physical_nopu: 4.5909 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3430 | Classification_binary: 2.9618 | Regression_cos_phi: 1.7051 | Regression_energy: 1.4836 | Regression_eta: 1.2330 | Regression_pt: 1.8586 | Regression_sin_phi: 1.6327 | LR: 3.12e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.8907s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0027 | max_forward: 0.1127 | max_loss: 0.0150 | max_backward: 0.3396 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.4660 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5295.8 | uss_MB: 4291.2 | child_rss_MB: 9057.9 | allocated_MB: 404.8 | reserved_MB: 11922.0 | inactive_split_MB: 135.4 +rank5 [2026-10-27 18:07:50,121] INFO : Step 34000/100000 rank5 | Train Loss: 14.2400 | Train Loss Unweighted: 4.7634 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5051 | Regression_cos_phi: 0.0450 | Regression_energy: 0.5627 | Regression_eta: 0.0631 | Regression_pt: 0.5251 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7180 | physical_nopu: 4.5858 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3294 | Classification_binary: 3.5642 | Regression_cos_phi: 1.8021 | Regression_energy: 2.7622 | Regression_eta: 0.6621 | Regression_pt: 3.7610 | Regression_sin_phi: 1.7827 | LR: 3.07e-04 | DataLoad Time: 0.0012s | Model Forward Time: 2.7415s | Phase Times: max_pre_step_barrier: 0.1802 | max_batch_to_device: 0.0765 | max_forward: 0.1650 | max_loss: 0.0036 | max_backward: 0.0935 | max_optimizer_step: 0.2788 | max_tensorboard_logging: 1.2621 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5285.5 | uss_MB: 4563.5 | child_rss_MB: 9059.3 | allocated_MB: 395.5 | reserved_MB: 11922.0 | inactive_split_MB: 99.3 +rank5 [2026-10-27 18:10:53,637] INFO : Step 35000/100000 rank5 | Train Loss: 11.4005 | Train Loss Unweighted: 4.4142 | Train Loss Components: Classification: 0.0091 | Classification_binary: 3.6972 | Regression_cos_phi: 0.0530 | Regression_energy: 0.2984 | Regression_eta: 0.0671 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.7147 | physical_nopu: 4.5802 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2128 | Classification_binary: 3.8586 | Regression_cos_phi: 1.8431 | Regression_energy: 1.4735 | Regression_eta: 0.8205 | Regression_pt: 1.5967 | Regression_sin_phi: 2.3764 | LR: 3.02e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4160s | Phase Times: max_pre_step_barrier: 0.0682 | max_batch_to_device: 0.0008 | max_forward: 0.1243 | max_loss: 0.0038 | max_backward: 0.0883 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.1949 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5274.5 | uss_MB: 4566.4 | child_rss_MB: 9063.3 | allocated_MB: 393.8 | reserved_MB: 11922.0 | inactive_split_MB: 85.2 +rank5 [2026-10-27 18:13:50,577] INFO : Step 36000/100000 rank5 | Train Loss: 11.8438 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.5751 | Regression_cos_phi: 0.0431 | Regression_energy: 0.3555 | Regression_eta: 0.0641 | Regression_pt: 0.3210 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.7110 | physical_nopu: 4.5729 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4084 | Classification_binary: 3.5487 | Regression_cos_phi: 1.4825 | Regression_energy: 1.6440 | Regression_eta: 0.6828 | Regression_pt: 2.1310 | Regression_sin_phi: 1.6730 | LR: 2.97e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.4515s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0993 | max_loss: 0.0066 | max_backward: 0.1760 | max_optimizer_step: 0.1735 | max_tensorboard_logging: 1.2166 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 5259.5 | uss_MB: 4568.3 | child_rss_MB: 9032.9 | allocated_MB: 395.0 | reserved_MB: 11922.0 | inactive_split_MB: 95.2 +rank5 [2026-10-27 18:16:45,596] INFO : Step 37000/100000 rank5 | Train Loss: 15.1390 | Train Loss Unweighted: 4.8166 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4396 | Regression_cos_phi: 0.0489 | Regression_energy: 0.6142 | Regression_eta: 0.0886 | Regression_pt: 0.5621 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7102 | physical_nopu: 4.5673 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3352 | Classification_binary: 3.4099 | Regression_cos_phi: 1.5132 | Regression_energy: 2.8624 | Regression_eta: 0.6101 | Regression_pt: 4.1336 | Regression_sin_phi: 1.2866 | LR: 2.91e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.2786s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1889 | max_loss: 0.0036 | max_backward: 0.0899 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 1.0901 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 5243.2 | uss_MB: 4569.9 | child_rss_MB: 9022.4 | allocated_MB: 394.3 | reserved_MB: 11922.0 | inactive_split_MB: 107.1 +rank5 [2026-10-27 18:19:42,075] INFO : Step 38000/100000 rank5 | Train Loss: 11.6256 | Train Loss Unweighted: 4.4424 | Train Loss Components: Classification: 0.0117 | Classification_binary: 3.6501 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3369 | Regression_eta: 0.0664 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7131 | physical_nopu: 4.5682 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2551 | Classification_binary: 3.5677 | Regression_cos_phi: 2.0739 | Regression_energy: 2.2145 | Regression_eta: 0.6368 | Regression_pt: 2.8416 | Regression_sin_phi: 1.4058 | LR: 2.86e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.2500s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.0965 | max_loss: 0.0038 | max_backward: 0.1051 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5237.2 | uss_MB: 4571.0 | child_rss_MB: 9030.3 | allocated_MB: 393.3 | reserved_MB: 12082.0 | inactive_split_MB: 110.5 +rank5 [2026-10-27 18:22:37,812] INFO : Step 39000/100000 rank5 | Train Loss: 13.0208 | Train Loss Unweighted: 4.4928 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4758 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4360 | Regression_eta: 0.0793 | Regression_pt: 0.3879 | Regression_sin_phi: 0.0500 | Group Train Loss Unweighted (window mean): physical_pu: 3.7060 | physical_nopu: 4.5607 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4074 | Classification_binary: 3.6396 | Regression_cos_phi: 1.9300 | Regression_energy: 2.0930 | Regression_eta: 0.8657 | Regression_pt: 2.6527 | Regression_sin_phi: 1.7088 | LR: 2.81e-04 | DataLoad Time: 0.0015s | Model Forward Time: 2.2317s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.0864 | max_loss: 0.0038 | max_backward: 0.0877 | max_optimizer_step: 0.1700 | max_tensorboard_logging: 1.1418 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5224.7 | uss_MB: 4572.8 | child_rss_MB: 9008.2 | allocated_MB: 394.4 | reserved_MB: 12706.0 | inactive_split_MB: 84.9 +rank5 [2026-10-27 18:25:31,277] INFO : Step 40000/100000 rank5 | Train Loss: 14.7562 | Train Loss Unweighted: 4.9007 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5708 | Regression_cos_phi: 0.0459 | Regression_energy: 0.5954 | Regression_eta: 0.0747 | Regression_pt: 0.5553 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7006 | physical_nopu: 4.5579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3860 | Classification_binary: 3.3915 | Regression_cos_phi: 1.5448 | Regression_energy: 6.5713 | Regression_eta: 0.6516 | Regression_pt: 9.3586 | Regression_sin_phi: 1.4183 | LR: 2.76e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.3072s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.0814 | max_loss: 0.0614 | max_backward: 0.1417 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 1.1378 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5222.7 | uss_MB: 4573.6 | child_rss_MB: 8992.8 | allocated_MB: 393.7 | reserved_MB: 12706.0 | inactive_split_MB: 107.7 +rank5 [2026-10-27 18:25:32,321] INFO : Running validation on rank5 +rank5 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 18:27:23,860] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 18:27:24,220] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 18:27:48,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 18:27:49,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 18:27:49,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 18:27:50,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 18:27:50,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 18:27:50,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 18:27:51,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 18:27:51,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 18:28:01,865] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 18:28:01,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 18:28:02,236] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 18:28:17,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 18:28:17,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 18:28:18,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 18:28:18,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 18:28:19,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 18:28:19,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 18:28:20,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 18:28:20,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 18:28:30,533] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 18:28:30,989] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 18:28:31,367] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 18:28:46,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 18:28:46,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 18:28:47,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 18:28:47,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 18:28:48,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 18:28:48,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 18:28:49,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 18:28:49,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 18:28:59,551] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 18:32:27,442] INFO : Step 41000/100000 rank5 | Train Loss: 12.3609 | Train Loss Unweighted: 4.4817 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5838 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3837 | Regression_eta: 0.0731 | Regression_pt: 0.3287 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.7016 | physical_nopu: 4.5433 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3303 | Classification_binary: 3.5304 | Regression_cos_phi: 2.3530 | Regression_energy: 1.6815 | Regression_eta: 0.9015 | Regression_pt: 2.0722 | Regression_sin_phi: 1.7197 | LR: 2.70e-04 | DataLoad Time: 0.0019s | Model Forward Time: 4.5172s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0009 | max_forward: 0.1338 | max_loss: 0.0038 | max_backward: 0.1857 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 3.1120 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 7028.8 | uss_MB: 6392.1 | child_rss_MB: 8998.3 | allocated_MB: 394.0 | reserved_MB: 11364.0 | inactive_split_MB: 106.1 +rank5 [2026-10-27 18:35:19,857] INFO : Step 42000/100000 rank5 | Train Loss: 12.9790 | Train Loss Unweighted: 4.5354 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4379 | Regression_cos_phi: 0.0458 | Regression_energy: 0.4856 | Regression_eta: 0.0625 | Regression_pt: 0.4500 | Regression_sin_phi: 0.0370 | Group Train Loss Unweighted (window mean): physical_pu: 3.6959 | physical_nopu: 4.5474 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4797 | Classification_binary: 3.4006 | Regression_cos_phi: 1.9997 | Regression_energy: 3.6473 | Regression_eta: 0.8568 | Regression_pt: 5.2749 | Regression_sin_phi: 1.1084 | LR: 2.65e-04 | DataLoad Time: 0.0120s | Model Forward Time: 2.6320s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0009 | max_forward: 0.1375 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 1.3492 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 7016.8 | uss_MB: 5332.4 | child_rss_MB: 8994.4 | allocated_MB: 394.2 | reserved_MB: 12150.0 | inactive_split_MB: 99.3 +rank5 [2026-10-27 18:38:35,926] INFO : Step 43000/100000 rank5 | Train Loss: 10.9006 | Train Loss Unweighted: 3.7057 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.9740 | Regression_cos_phi: 0.0473 | Regression_energy: 0.2786 | Regression_eta: 0.1094 | Regression_pt: 0.2322 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6942 | physical_nopu: 4.5421 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3788 | Classification_binary: 3.0178 | Regression_cos_phi: 1.6468 | Regression_energy: 1.3499 | Regression_eta: 1.2454 | Regression_pt: 1.6142 | Regression_sin_phi: 1.5419 | LR: 2.59e-04 | DataLoad Time: 0.0020s | Model Forward Time: 5.2293s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0039 | max_forward: 0.1100 | max_loss: 0.0637 | max_backward: 0.2303 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 3.9250 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5285.0 | uss_MB: 4686.1 | child_rss_MB: 8875.5 | allocated_MB: 406.8 | reserved_MB: 12150.0 | inactive_split_MB: 152.7 +rank5 [2026-10-27 18:41:39,775] INFO : Step 44000/100000 rank5 | Train Loss: 14.8629 | Train Loss Unweighted: 4.8399 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4856 | Regression_cos_phi: 0.0438 | Regression_energy: 0.6070 | Regression_eta: 0.0739 | Regression_pt: 0.5644 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6933 | physical_nopu: 4.5331 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.5284 | Classification_binary: 3.4901 | Regression_cos_phi: 1.7430 | Regression_energy: 2.9102 | Regression_eta: 0.9810 | Regression_pt: 3.4501 | Regression_sin_phi: 2.4614 | LR: 2.54e-04 | DataLoad Time: 0.0012s | Model Forward Time: 2.6846s | Phase Times: max_pre_step_barrier: 0.0851 | max_batch_to_device: 0.0008 | max_forward: 0.0946 | max_loss: 0.0041 | max_backward: 0.0964 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.3558 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5284.0 | uss_MB: 4679.1 | child_rss_MB: 8885.5 | allocated_MB: 394.8 | reserved_MB: 12150.0 | inactive_split_MB: 94.9 +rank5 [2026-10-27 18:44:46,984] INFO : Step 45000/100000 rank5 | Train Loss: 12.9535 | Train Loss Unweighted: 4.5658 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.6115 | Regression_cos_phi: 0.0537 | Regression_energy: 0.4117 | Regression_eta: 0.0846 | Regression_pt: 0.3413 | Regression_sin_phi: 0.0531 | Group Train Loss Unweighted (window mean): physical_pu: 3.6892 | physical_nopu: 4.5283 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2542 | Classification_binary: 3.4731 | Regression_cos_phi: 2.2031 | Regression_energy: 2.8176 | Regression_eta: 2.2800 | Regression_pt: 3.2155 | Regression_sin_phi: 2.8533 | LR: 2.48e-04 | DataLoad Time: 0.0029s | Model Forward Time: 2.4261s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1077 | max_loss: 0.0040 | max_backward: 0.1457 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 1.2404 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 5275.4 | uss_MB: 4680.1 | child_rss_MB: 8893.2 | allocated_MB: 394.0 | reserved_MB: 13366.0 | inactive_split_MB: 74.2 +rank5 [2026-10-27 18:48:06,241] INFO : Step 46000/100000 rank5 | Train Loss: 12.3783 | Train Loss Unweighted: 4.4476 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.5252 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3923 | Regression_eta: 0.0679 | Regression_pt: 0.3506 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6869 | physical_nopu: 4.5281 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3380 | Classification_binary: 3.5455 | Regression_cos_phi: 1.9064 | Regression_energy: 1.9139 | Regression_eta: 0.6889 | Regression_pt: 2.5861 | Regression_sin_phi: 2.0398 | LR: 2.43e-04 | DataLoad Time: 0.0014s | Model Forward Time: 2.4378s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1021 | max_loss: 0.0036 | max_backward: 0.1895 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 1.1894 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 5269.4 | uss_MB: 4681.7 | child_rss_MB: 8897.5 | allocated_MB: 395.1 | reserved_MB: 13366.0 | inactive_split_MB: 103.1 +rank5 [2026-10-27 18:51:16,142] INFO : Step 47000/100000 rank5 | Train Loss: 14.5142 | Train Loss Unweighted: 4.7631 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.3922 | Regression_cos_phi: 0.0407 | Regression_energy: 0.6220 | Regression_eta: 0.0662 | Regression_pt: 0.5862 | Regression_sin_phi: 0.0406 | Group Train Loss Unweighted (window mean): physical_pu: 3.6853 | physical_nopu: 4.5234 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2442 | Classification_binary: 3.2946 | Regression_cos_phi: 1.7407 | Regression_energy: 3.0404 | Regression_eta: 0.8544 | Regression_pt: 3.8989 | Regression_sin_phi: 1.7650 | LR: 2.37e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.5380s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0667 | max_forward: 0.1426 | max_loss: 0.0037 | max_backward: 0.0873 | max_optimizer_step: 0.2591 | max_tensorboard_logging: 1.1650 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5267.4 | uss_MB: 4682.3 | child_rss_MB: 8896.3 | allocated_MB: 394.9 | reserved_MB: 14166.0 | inactive_split_MB: 95.2 +rank5 [2026-10-27 18:54:12,871] INFO : Step 48000/100000 rank5 | Train Loss: 11.2030 | Train Loss Unweighted: 4.3343 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.6447 | Regression_cos_phi: 0.0521 | Regression_energy: 0.2822 | Regression_eta: 0.0657 | Regression_pt: 0.2293 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6844 | physical_nopu: 4.5168 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2066 | Classification_binary: 3.5985 | Regression_cos_phi: 1.2262 | Regression_energy: 1.3176 | Regression_eta: 0.5700 | Regression_pt: 1.6307 | Regression_sin_phi: 1.5498 | LR: 2.31e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.2610s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.0058 | max_backward: 0.0852 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1613 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5253.4 | uss_MB: 4683.3 | child_rss_MB: 8890.9 | allocated_MB: 393.9 | reserved_MB: 14166.0 | inactive_split_MB: 101.2 +rank5 [2026-10-27 18:57:19,782] INFO : Step 49000/100000 rank5 | Train Loss: 12.4372 | Train Loss Unweighted: 4.4483 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.5320 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3881 | Regression_eta: 0.0742 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6801 | physical_nopu: 4.5174 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3843 | Classification_binary: 3.5514 | Regression_cos_phi: 1.6445 | Regression_energy: 1.7771 | Regression_eta: 0.5275 | Regression_pt: 2.3379 | Regression_sin_phi: 1.4791 | LR: 2.26e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4741s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0009 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1014 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 1.3340 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5240.2 | uss_MB: 4683.4 | child_rss_MB: 8815.5 | allocated_MB: 393.7 | reserved_MB: 14166.0 | inactive_split_MB: 100.8 +rank5 [2026-10-27 19:00:32,233] INFO : Step 50000/100000 rank5 | Train Loss: 13.3654 | Train Loss Unweighted: 4.5742 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4424 | Regression_cos_phi: 0.0426 | Regression_energy: 0.5013 | Regression_eta: 0.0683 | Regression_pt: 0.4573 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6814 | physical_nopu: 4.5132 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3044 | Classification_binary: 3.5037 | Regression_cos_phi: 1.4048 | Regression_energy: 3.0853 | Regression_eta: 0.6888 | Regression_pt: 4.1953 | Regression_sin_phi: 1.3425 | LR: 2.20e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.5859s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0010 | max_forward: 0.1892 | max_loss: 0.0115 | max_backward: 0.1333 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.2641 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0862 | Memory: rss_MB: 5234.2 | uss_MB: 4685.5 | child_rss_MB: 8779.3 | allocated_MB: 394.6 | reserved_MB: 14166.0 | inactive_split_MB: 91.2 +rank5 [2026-10-27 19:00:33,174] INFO : Running validation on rank5 +rank5 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 19:02:37,502] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 19:02:37,941] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 19:03:02,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 19:03:03,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 19:03:03,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 19:03:04,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 19:03:04,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 19:03:05,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 19:03:05,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 19:03:06,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 19:03:16,117] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 19:03:16,262] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 19:03:16,635] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 19:03:31,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 19:03:32,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 19:03:32,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 19:03:33,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 19:03:33,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 19:03:34,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 19:03:34,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 19:03:35,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 19:03:45,128] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 19:03:45,439] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 19:03:45,816] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 19:04:01,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 19:04:01,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 19:04:02,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 19:04:02,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 19:04:03,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 19:04:03,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 19:04:03,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 19:04:04,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 19:04:14,342] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-27 19:07:44,424] INFO : Step 51000/100000 rank5 | Train Loss: 12.0993 | Train Loss Unweighted: 4.3946 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5485 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3591 | Regression_eta: 0.0789 | Regression_pt: 0.2941 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6745 | physical_nopu: 4.5037 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2940 | Classification_binary: 3.6086 | Regression_cos_phi: 1.7806 | Regression_energy: 1.2908 | Regression_eta: 0.5973 | Regression_pt: 1.6374 | Regression_sin_phi: 1.4827 | LR: 2.14e-04 | DataLoad Time: 0.0099s | Model Forward Time: 3.3975s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0008 | max_forward: 0.1391 | max_loss: 0.0036 | max_backward: 0.1509 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 2.0683 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 7102.6 | uss_MB: 6555.6 | child_rss_MB: 8775.1 | allocated_MB: 393.6 | reserved_MB: 10952.0 | inactive_split_MB: 101.2 +rank5 [2026-10-27 19:10:55,345] INFO : Step 52000/100000 rank5 | Train Loss: 10.7195 | Train Loss Unweighted: 3.6688 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9553 | Regression_cos_phi: 0.0470 | Regression_energy: 0.2719 | Regression_eta: 0.1059 | Regression_pt: 0.2258 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6784 | physical_nopu: 4.5078 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3344 | Classification_binary: 2.9509 | Regression_cos_phi: 1.7009 | Regression_energy: 1.3613 | Regression_eta: 1.1788 | Regression_pt: 1.6385 | Regression_sin_phi: 1.6312 | LR: 2.09e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.5037s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0031 | max_forward: 0.1853 | max_loss: 0.0256 | max_backward: 0.2604 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 2.0239 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7092.6 | uss_MB: 6564.5 | child_rss_MB: 8773.5 | allocated_MB: 404.8 | reserved_MB: 11626.0 | inactive_split_MB: 256.4 +rank5 [2026-10-27 19:13:48,854] INFO : Step 53000/100000 rank5 | Train Loss: 11.7022 | Train Loss Unweighted: 4.3546 | Train Loss Components: Classification: 0.0123 | Classification_binary: 3.5594 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3321 | Regression_eta: 0.0730 | Regression_pt: 0.2795 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6748 | physical_nopu: 4.5005 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3051 | Classification_binary: 3.5361 | Regression_cos_phi: 1.8408 | Regression_energy: 1.3829 | Regression_eta: 0.8274 | Regression_pt: 1.5654 | Regression_sin_phi: 1.6435 | LR: 2.03e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.1499s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0954 | max_loss: 0.0037 | max_backward: 0.1065 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.9892 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 5255.7 | uss_MB: 4729.4 | child_rss_MB: 8773.6 | allocated_MB: 393.6 | reserved_MB: 11910.0 | inactive_split_MB: 100.6 +rank5 [2026-10-27 19:17:18,773] INFO : Step 54000/100000 rank5 | Train Loss: 12.5762 | Train Loss Unweighted: 4.3665 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0487 | Regression_energy: 0.4104 | Regression_eta: 0.0739 | Regression_pt: 0.3685 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6696 | physical_nopu: 4.5036 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3356 | Classification_binary: 3.4461 | Regression_cos_phi: 1.5045 | Regression_energy: 2.1699 | Regression_eta: 0.6291 | Regression_pt: 3.0576 | Regression_sin_phi: 1.6040 | LR: 1.97e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.3957s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.1022 | max_forward: 0.1953 | max_loss: 0.0037 | max_backward: 0.0975 | max_optimizer_step: 0.2772 | max_tensorboard_logging: 2.0527 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5248.7 | uss_MB: 4730.3 | child_rss_MB: 8744.8 | allocated_MB: 394.9 | reserved_MB: 11910.0 | inactive_split_MB: 118.2 +rank5 [2026-10-27 19:20:28,222] INFO : Step 55000/100000 rank5 | Train Loss: 13.4632 | Train Loss Unweighted: 4.6500 | Train Loss Components: Classification: 0.0134 | Classification_binary: 3.5428 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4921 | Regression_eta: 0.0679 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6720 | physical_nopu: 4.4919 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4713 | Classification_binary: 3.5028 | Regression_cos_phi: 1.5831 | Regression_energy: 3.4871 | Regression_eta: 0.6499 | Regression_pt: 4.9098 | Regression_sin_phi: 1.1546 | LR: 1.92e-04 | DataLoad Time: 0.0290s | Model Forward Time: 3.7751s | Phase Times: max_pre_step_barrier: 0.1980 | max_batch_to_device: 0.0007 | max_forward: 0.0889 | max_loss: 0.0037 | max_backward: 0.2091 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 2.3779 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5235.8 | uss_MB: 4729.6 | child_rss_MB: 8640.7 | allocated_MB: 393.8 | reserved_MB: 11910.0 | inactive_split_MB: 89.1 +rank5 [2026-10-27 19:23:30,605] INFO : Step 56000/100000 rank5 | Train Loss: 11.6078 | Train Loss Unweighted: 4.2962 | Train Loss Components: Classification: 0.0140 | Classification_binary: 3.4759 | Regression_cos_phi: 0.0438 | Regression_energy: 0.3484 | Regression_eta: 0.0671 | Regression_pt: 0.2977 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6692 | physical_nopu: 4.4908 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3559 | Classification_binary: 3.4369 | Regression_cos_phi: 1.5247 | Regression_energy: 1.2668 | Regression_eta: 0.5642 | Regression_pt: 1.6035 | Regression_sin_phi: 1.3708 | LR: 1.86e-04 | DataLoad Time: 0.0934s | Model Forward Time: 3.1380s | Phase Times: max_pre_step_barrier: 0.0941 | max_batch_to_device: 0.0008 | max_forward: 0.0930 | max_loss: 0.0038 | max_backward: 0.0988 | max_optimizer_step: 0.1756 | max_tensorboard_logging: 1.8920 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 5221.8 | uss_MB: 4732.9 | child_rss_MB: 8582.9 | allocated_MB: 393.5 | reserved_MB: 11910.0 | inactive_split_MB: 115.7 +rank5 [2026-10-27 19:26:52,902] INFO : Step 57000/100000 rank5 | Train Loss: 13.9252 | Train Loss Unweighted: 4.6468 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4640 | Regression_cos_phi: 0.0464 | Regression_energy: 0.5144 | Regression_eta: 0.0781 | Regression_pt: 0.4791 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6687 | physical_nopu: 4.4875 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4329 | Classification_binary: 3.3811 | Regression_cos_phi: 1.8856 | Regression_energy: 2.0919 | Regression_eta: 1.2297 | Regression_pt: 2.8255 | Regression_sin_phi: 1.8031 | LR: 1.81e-04 | DataLoad Time: 0.0028s | Model Forward Time: 4.6255s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.1223 | max_forward: 0.2094 | max_loss: 0.0420 | max_backward: 0.1628 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 2.9072 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 5216.8 | uss_MB: 4734.2 | child_rss_MB: 8511.2 | allocated_MB: 394.3 | reserved_MB: 12240.0 | inactive_split_MB: 101.4 +rank5 [2026-10-27 19:30:07,651] INFO : Step 58000/100000 rank5 | Train Loss: 11.5910 | Train Loss Unweighted: 4.3047 | Train Loss Components: Classification: 0.0106 | Classification_binary: 3.5490 | Regression_cos_phi: 0.0544 | Regression_energy: 0.3108 | Regression_eta: 0.0631 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0510 | Group Train Loss Unweighted (window mean): physical_pu: 3.6630 | physical_nopu: 4.4842 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1679 | Classification_binary: 3.3981 | Regression_cos_phi: 2.1300 | Regression_energy: 1.3167 | Regression_eta: 0.7537 | Regression_pt: 1.7509 | Regression_sin_phi: 1.5583 | LR: 1.75e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.5120s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.1080 | max_backward: 0.1964 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.2749 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 5209.3 | uss_MB: 4734.4 | child_rss_MB: 8432.6 | allocated_MB: 393.9 | reserved_MB: 12240.0 | inactive_split_MB: 121.9 +rank5 [2026-10-27 19:33:12,149] INFO : Step 59000/100000 rank5 | Train Loss: 12.2218 | Train Loss Unweighted: 4.4115 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5166 | Regression_cos_phi: 0.0458 | Regression_energy: 0.3794 | Regression_eta: 0.0771 | Regression_pt: 0.3280 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6650 | physical_nopu: 4.4788 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3792 | Classification_binary: 3.5752 | Regression_cos_phi: 1.8236 | Regression_energy: 1.8882 | Regression_eta: 0.7688 | Regression_pt: 2.2687 | Regression_sin_phi: 1.6134 | LR: 1.70e-04 | DataLoad Time: 0.6884s | Model Forward Time: 2.5342s | Phase Times: max_pre_step_barrier: 0.6889 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0499 | max_backward: 0.1549 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5201.3 | uss_MB: 4735.4 | child_rss_MB: 8387.3 | allocated_MB: 394.0 | reserved_MB: 12240.0 | inactive_split_MB: 135.3 +rank5 [2026-10-27 19:36:26,517] INFO : Step 60000/100000 rank5 | Train Loss: 14.3810 | Train Loss Unweighted: 4.7229 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.4146 | Regression_cos_phi: 0.0449 | Regression_energy: 0.5841 | Regression_eta: 0.0716 | Regression_pt: 0.5495 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.6604 | physical_nopu: 4.4865 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3178 | Classification_binary: 3.5034 | Regression_cos_phi: 1.9495 | Regression_energy: 2.1962 | Regression_eta: 0.6763 | Regression_pt: 2.8195 | Regression_sin_phi: 1.5462 | LR: 1.64e-04 | DataLoad Time: 0.0029s | Model Forward Time: 2.4562s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0849 | max_loss: 0.0036 | max_backward: 0.1478 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 1.2880 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5191.9 | uss_MB: 4495.0 | child_rss_MB: 8328.9 | allocated_MB: 394.8 | reserved_MB: 12240.0 | inactive_split_MB: 87.0 +rank5 [2026-10-27 19:36:27,558] INFO : Running validation on rank5 +rank5 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 19:38:18,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 19:38:19,029] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 19:38:43,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 19:38:44,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 19:38:44,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 19:38:45,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 19:38:45,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 19:38:46,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 19:38:46,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 19:38:46,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 19:38:56,979] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 19:38:57,241] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 19:38:57,609] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 19:39:12,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 19:39:13,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 19:39:13,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 19:39:14,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 19:39:14,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 19:39:15,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 19:39:15,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 19:39:15,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 19:39:26,007] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 19:39:26,170] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 19:39:26,533] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 19:39:41,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 19:39:42,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 19:39:42,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 19:39:43,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 19:39:43,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 19:39:44,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 19:39:44,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 19:39:44,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 19:39:54,855] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 19:43:33,544] INFO : Step 61000/100000 rank5 | Train Loss: 11.4262 | Train Loss Unweighted: 4.3706 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.6500 | Regression_cos_phi: 0.0531 | Regression_energy: 0.2956 | Regression_eta: 0.0675 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6553 | physical_nopu: 4.4780 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3070 | Classification_binary: 3.6775 | Regression_cos_phi: 1.7009 | Regression_energy: 1.4913 | Regression_eta: 0.6696 | Regression_pt: 1.8281 | Regression_sin_phi: 1.6016 | LR: 1.59e-04 | DataLoad Time: 0.0069s | Model Forward Time: 3.1995s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1116 | max_loss: 0.0036 | max_backward: 0.1506 | max_optimizer_step: 0.1828 | max_tensorboard_logging: 1.9738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 5784.0 | uss_MB: 5329.2 | child_rss_MB: 8302.4 | allocated_MB: 393.5 | reserved_MB: 11190.0 | inactive_split_MB: 103.1 +rank5 [2026-10-27 19:46:49,589] INFO : Step 62000/100000 rank5 | Train Loss: 10.7781 | Train Loss Unweighted: 3.6510 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9253 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2791 | Regression_eta: 0.1043 | Regression_pt: 0.2316 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3542 | Classification_binary: 2.8402 | Regression_cos_phi: 1.6630 | Regression_energy: 1.2451 | Regression_eta: 1.0590 | Regression_pt: 1.5147 | Regression_sin_phi: 1.5611 | LR: 1.54e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.6555s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0030 | max_forward: 0.1294 | max_loss: 0.0038 | max_backward: 0.2007 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.2115 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0496 | Memory: rss_MB: 5775.0 | uss_MB: 5052.2 | child_rss_MB: 8186.6 | allocated_MB: 404.4 | reserved_MB: 11190.0 | inactive_split_MB: 148.6 +rank5 [2026-10-27 19:50:02,443] INFO : Step 63000/100000 rank5 | Train Loss: 11.5766 | Train Loss Unweighted: 4.3655 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6000 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3204 | Regression_eta: 0.0727 | Regression_pt: 0.2610 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6589 | physical_nopu: 4.4771 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1823 | Classification_binary: 3.5980 | Regression_cos_phi: 1.9077 | Regression_energy: 1.7500 | Regression_eta: 0.8290 | Regression_pt: 2.0984 | Regression_sin_phi: 1.9839 | LR: 1.49e-04 | DataLoad Time: 0.0015s | Model Forward Time: 2.3897s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.1036 | max_loss: 0.0037 | max_backward: 0.1140 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2208 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 5773.0 | uss_MB: 5331.6 | child_rss_MB: 8138.8 | allocated_MB: 393.8 | reserved_MB: 11190.0 | inactive_split_MB: 93.0 +rank5 [2026-10-27 19:53:17,771] INFO : Step 64000/100000 rank5 | Train Loss: 13.2709 | Train Loss Unweighted: 4.5542 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.4295 | Regression_cos_phi: 0.0429 | Regression_energy: 0.4948 | Regression_eta: 0.0672 | Regression_pt: 0.4587 | Regression_sin_phi: 0.0436 | Group Train Loss Unweighted (window mean): physical_pu: 3.6551 | physical_nopu: 4.4714 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3781 | Classification_binary: 3.3550 | Regression_cos_phi: 1.4654 | Regression_energy: 2.4795 | Regression_eta: 0.6050 | Regression_pt: 3.3226 | Regression_sin_phi: 1.6080 | LR: 1.43e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.3964s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0928 | max_loss: 0.0570 | max_backward: 0.1589 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 1.2035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 5289.6 | uss_MB: 4857.6 | child_rss_MB: 8120.4 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank5 [2026-10-27 19:56:23,053] INFO : Step 65000/100000 rank5 | Train Loss: 15.6652 | Train Loss Unweighted: 4.9833 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5112 | Regression_cos_phi: 0.0496 | Regression_energy: 0.6688 | Regression_eta: 0.0629 | Regression_pt: 0.6290 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6554 | physical_nopu: 4.4650 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3231 | Classification_binary: 3.5732 | Regression_cos_phi: 1.7500 | Regression_energy: 6.2293 | Regression_eta: 0.7762 | Regression_pt: 8.8176 | Regression_sin_phi: 1.7862 | LR: 1.38e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.7628s | Phase Times: max_pre_step_barrier: 0.1158 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0042 | max_backward: 0.1974 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.4176 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5282.6 | uss_MB: 4858.3 | child_rss_MB: 8025.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 124.9 +rank5 [2026-10-27 19:59:28,722] INFO : Step 66000/100000 rank5 | Train Loss: 12.0073 | Train Loss Unweighted: 4.3350 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4949 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3535 | Regression_eta: 0.0691 | Regression_pt: 0.3018 | Regression_sin_phi: 0.0501 | Group Train Loss Unweighted (window mean): physical_pu: 3.6526 | physical_nopu: 4.4657 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3396 | Classification_binary: 3.4998 | Regression_cos_phi: 2.2423 | Regression_energy: 1.5775 | Regression_eta: 0.5802 | Regression_pt: 2.0597 | Regression_sin_phi: 2.3056 | LR: 1.33e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.4472s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0891 | max_loss: 0.0538 | max_backward: 0.1468 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 1.2753 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5277.7 | uss_MB: 4858.9 | child_rss_MB: 7982.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 107.1 +rank5 [2026-10-27 20:02:51,910] INFO : Step 67000/100000 rank5 | Train Loss: 13.3177 | Train Loss Unweighted: 4.5554 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.4800 | Regression_cos_phi: 0.0500 | Regression_energy: 0.4690 | Regression_eta: 0.0673 | Regression_pt: 0.4253 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6538 | physical_nopu: 4.4665 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3431 | Classification_binary: 3.3801 | Regression_cos_phi: 1.6463 | Regression_energy: 1.8768 | Regression_eta: 0.5163 | Regression_pt: 2.6140 | Regression_sin_phi: 1.4674 | LR: 1.28e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.9379s | Phase Times: max_pre_step_barrier: 0.0792 | max_batch_to_device: 0.0009 | max_forward: 0.0886 | max_loss: 0.1057 | max_backward: 0.1987 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 2.6109 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 5271.7 | uss_MB: 4859.8 | child_rss_MB: 7952.8 | allocated_MB: 393.9 | reserved_MB: 11792.0 | inactive_split_MB: 111.5 +rank5 [2026-10-27 20:06:09,103] INFO : Step 68000/100000 rank5 | Train Loss: 11.1535 | Train Loss Unweighted: 4.2643 | Train Loss Components: Classification: 0.0109 | Classification_binary: 3.5138 | Regression_cos_phi: 0.0456 | Regression_energy: 0.3163 | Regression_eta: 0.0648 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6527 | physical_nopu: 4.4693 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2193 | Classification_binary: 3.4941 | Regression_cos_phi: 1.9714 | Regression_energy: 1.8214 | Regression_eta: 0.9905 | Regression_pt: 2.2087 | Regression_sin_phi: 1.9922 | LR: 1.24e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.3895s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1008 | max_loss: 0.0767 | max_backward: 0.1768 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.1759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5263.4 | uss_MB: 4860.3 | child_rss_MB: 7883.7 | allocated_MB: 394.5 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank5 [2026-10-27 20:09:12,492] INFO : Step 69000/100000 rank5 | Train Loss: 12.1728 | Train Loss Unweighted: 4.3418 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4719 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3660 | Regression_eta: 0.0763 | Regression_pt: 0.3115 | Regression_sin_phi: 0.0548 | Group Train Loss Unweighted (window mean): physical_pu: 3.6494 | physical_nopu: 4.4629 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3840 | Classification_binary: 3.5007 | Regression_cos_phi: 1.8130 | Regression_energy: 1.5450 | Regression_eta: 1.1592 | Regression_pt: 2.0597 | Regression_sin_phi: 1.7164 | LR: 1.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.2959s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.1094 | max_loss: 0.0147 | max_backward: 0.1167 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 1.1204 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5251.4 | uss_MB: 4861.5 | child_rss_MB: 7816.8 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 106.5 +rank5 [2026-10-27 20:12:13,498] INFO : Step 70000/100000 rank5 | Train Loss: 14.9156 | Train Loss Unweighted: 4.7730 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.3975 | Regression_cos_phi: 0.0487 | Regression_energy: 0.6183 | Regression_eta: 0.0638 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0448 | Group Train Loss Unweighted (window mean): physical_pu: 3.6488 | physical_nopu: 4.4584 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3555 | Classification_binary: 3.4026 | Regression_cos_phi: 2.0924 | Regression_energy: 3.3279 | Regression_eta: 0.8785 | Regression_pt: 4.6850 | Regression_sin_phi: 1.7420 | LR: 1.14e-04 | DataLoad Time: 0.0074s | Model Forward Time: 2.3862s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1038 | max_loss: 0.0858 | max_backward: 0.1871 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 1.1619 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5245.8 | uss_MB: 4860.8 | child_rss_MB: 7775.8 | allocated_MB: 395.0 | reserved_MB: 11792.0 | inactive_split_MB: 111.9 +rank5 [2026-10-27 20:12:14,505] INFO : Running validation on rank5 +rank5 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 20:14:09,433] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 20:14:09,864] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 20:14:34,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 20:14:35,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 20:14:35,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 20:14:36,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 20:14:36,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 20:14:37,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 20:14:37,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 20:14:37,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 20:14:47,916] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 20:14:47,942] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 20:14:48,315] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 20:15:03,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 20:15:03,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 20:15:04,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 20:15:04,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 20:15:05,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 20:15:05,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 20:15:06,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 20:15:06,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 20:15:16,707] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 20:15:17,039] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 20:15:17,439] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 20:15:32,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 20:15:33,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 20:15:33,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 20:15:34,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 20:15:34,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 20:15:34,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 20:15:35,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 20:15:35,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 20:15:45,820] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 20:19:21,751] INFO : Step 71000/100000 rank5 | Train Loss: 10.9922 | Train Loss Unweighted: 3.6772 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9201 | Regression_cos_phi: 0.0477 | Regression_energy: 0.2906 | Regression_eta: 0.1066 | Regression_pt: 0.2485 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6484 | physical_nopu: 4.4561 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3385 | Classification_binary: 2.8287 | Regression_cos_phi: 1.6791 | Regression_energy: 1.3539 | Regression_eta: 1.1868 | Regression_pt: 1.7141 | Regression_sin_phi: 1.6163 | LR: 1.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.5358s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0652 | max_forward: 0.2596 | max_loss: 0.0036 | max_backward: 0.3497 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 1.8772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 7279.8 | uss_MB: 6900.2 | child_rss_MB: 7804.4 | allocated_MB: 405.9 | reserved_MB: 11130.0 | inactive_split_MB: 135.5 +rank5 [2026-10-27 20:22:34,777] INFO : Step 72000/100000 rank5 | Train Loss: 14.4733 | Train Loss Unweighted: 4.6898 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3667 | Regression_cos_phi: 0.0458 | Regression_energy: 0.5948 | Regression_eta: 0.0650 | Regression_pt: 0.5567 | Regression_sin_phi: 0.0444 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4536 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3374 | Classification_binary: 3.1560 | Regression_cos_phi: 1.9466 | Regression_energy: 3.9724 | Regression_eta: 0.5718 | Regression_pt: 5.5750 | Regression_sin_phi: 1.5293 | LR: 1.05e-04 | DataLoad Time: 0.0015s | Model Forward Time: 2.5073s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1390 | max_loss: 0.0037 | max_backward: 0.1007 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.2476 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1281 | Memory: rss_MB: 5316.2 | uss_MB: 4936.1 | child_rss_MB: 7727.0 | allocated_MB: 393.8 | reserved_MB: 11706.0 | inactive_split_MB: 88.5 +rank5 [2026-10-27 20:25:34,428] INFO : Step 73000/100000 rank5 | Train Loss: 11.7903 | Train Loss Unweighted: 4.2561 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.5010 | Regression_cos_phi: 0.0578 | Regression_energy: 0.3092 | Regression_eta: 0.0644 | Regression_pt: 0.2565 | Regression_sin_phi: 0.0557 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4567 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2122 | Classification_binary: 3.4858 | Regression_cos_phi: 1.7450 | Regression_energy: 1.4413 | Regression_eta: 0.6237 | Regression_pt: 1.8593 | Regression_sin_phi: 1.6189 | LR: 1.01e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.6064s | Phase Times: max_pre_step_barrier: 0.1246 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0122 | max_backward: 0.1972 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.2348 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5307.2 | uss_MB: 4937.3 | child_rss_MB: 7690.0 | allocated_MB: 393.5 | reserved_MB: 11706.0 | inactive_split_MB: 93.2 +rank5 [2026-10-27 20:28:47,370] INFO : Step 74000/100000 rank5 | Train Loss: 11.6748 | Train Loss Unweighted: 4.2803 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4309 | Regression_cos_phi: 0.0428 | Regression_energy: 0.3572 | Regression_eta: 0.0695 | Regression_pt: 0.3177 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.6485 | physical_nopu: 4.4502 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3439 | Classification_binary: 3.2977 | Regression_cos_phi: 1.5272 | Regression_energy: 1.9184 | Regression_eta: 0.6811 | Regression_pt: 2.4889 | Regression_sin_phi: 1.7117 | LR: 9.68e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.0484s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1195 | max_loss: 0.0060 | max_backward: 0.2082 | max_optimizer_step: 0.1763 | max_tensorboard_logging: 1.8023 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5300.9 | uss_MB: 4938.3 | child_rss_MB: 7614.5 | allocated_MB: 394.3 | reserved_MB: 11706.0 | inactive_split_MB: 97.0 +rank5 [2026-10-27 20:31:56,443] INFO : Step 75000/100000 rank5 | Train Loss: 14.1604 | Train Loss Unweighted: 4.6445 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.4007 | Regression_cos_phi: 0.0489 | Regression_energy: 0.5577 | Regression_eta: 0.0640 | Regression_pt: 0.5112 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.6489 | physical_nopu: 4.4484 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3366 | Classification_binary: 3.5216 | Regression_cos_phi: 1.9654 | Regression_energy: 2.2773 | Regression_eta: 0.6247 | Regression_pt: 2.9282 | Regression_sin_phi: 1.4667 | LR: 9.27e-05 | DataLoad Time: 0.0023s | Model Forward Time: 2.9329s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0831 | max_loss: 0.0038 | max_backward: 0.2058 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.5751 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1279 | Memory: rss_MB: 5290.9 | uss_MB: 4938.0 | child_rss_MB: 7581.3 | allocated_MB: 394.4 | reserved_MB: 11706.0 | inactive_split_MB: 89.1 +rank5 [2026-10-27 20:34:57,733] INFO : Step 76000/100000 rank5 | Train Loss: 11.4149 | Train Loss Unweighted: 4.2984 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5224 | Regression_cos_phi: 0.0478 | Regression_energy: 0.3298 | Regression_eta: 0.0709 | Regression_pt: 0.2685 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6454 | physical_nopu: 4.4486 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2691 | Classification_binary: 3.4653 | Regression_cos_phi: 2.2993 | Regression_energy: 2.0011 | Regression_eta: 1.8213 | Regression_pt: 2.1755 | Regression_sin_phi: 2.0617 | LR: 8.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 3.1736s | Phase Times: max_pre_step_barrier: 0.0296 | max_batch_to_device: 0.0996 | max_forward: 0.1816 | max_loss: 0.0036 | max_backward: 0.0930 | max_optimizer_step: 0.2829 | max_tensorboard_logging: 1.7104 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1287 | Memory: rss_MB: 5286.0 | uss_MB: 4937.3 | child_rss_MB: 7586.8 | allocated_MB: 393.6 | reserved_MB: 12330.0 | inactive_split_MB: 97.7 +rank5 [2026-10-27 20:38:16,884] INFO : Step 77000/100000 rank5 | Train Loss: 12.8789 | Train Loss Unweighted: 4.4653 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4333 | Regression_cos_phi: 0.0478 | Regression_energy: 0.4517 | Regression_eta: 0.0624 | Regression_pt: 0.4079 | Regression_sin_phi: 0.0461 | Group Train Loss Unweighted (window mean): physical_pu: 3.6433 | physical_nopu: 4.4448 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3377 | Classification_binary: 3.3499 | Regression_cos_phi: 1.7556 | Regression_energy: 2.5556 | Regression_eta: 0.9672 | Regression_pt: 3.4263 | Regression_sin_phi: 1.7110 | LR: 8.50e-05 | DataLoad Time: 0.0026s | Model Forward Time: 3.0356s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.0831 | max_loss: 0.0514 | max_backward: 0.1524 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.8652 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5273.0 | uss_MB: 4939.8 | child_rss_MB: 7463.9 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 99.2 +rank5 [2026-10-27 20:41:21,661] INFO : Step 78000/100000 rank5 | Train Loss: 13.2420 | Train Loss Unweighted: 4.6345 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.5815 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4598 | Regression_eta: 0.0679 | Regression_pt: 0.4144 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6415 | physical_nopu: 4.4494 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3530 | Classification_binary: 3.6195 | Regression_cos_phi: 1.5417 | Regression_energy: 3.0919 | Regression_eta: 0.7047 | Regression_pt: 4.3472 | Regression_sin_phi: 1.5143 | LR: 8.13e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.0719s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0927 | max_loss: 0.0036 | max_backward: 0.2060 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.8450 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5261.0 | uss_MB: 4940.8 | child_rss_MB: 7383.4 | allocated_MB: 394.7 | reserved_MB: 12330.0 | inactive_split_MB: 95.1 +rank5 [2026-10-27 20:44:31,405] INFO : Step 79000/100000 rank5 | Train Loss: 11.5047 | Train Loss Unweighted: 4.2761 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4919 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3261 | Regression_eta: 0.0640 | Regression_pt: 0.2834 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6416 | physical_nopu: 4.4460 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3073 | Classification_binary: 3.5063 | Regression_cos_phi: 1.5386 | Regression_energy: 1.4533 | Regression_eta: 0.6657 | Regression_pt: 1.7910 | Regression_sin_phi: 1.9615 | LR: 7.78e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.2158s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0253 | max_forward: 0.1473 | max_loss: 0.0038 | max_backward: 0.0955 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 1.9569 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5248.8 | uss_MB: 4513.6 | child_rss_MB: 7321.5 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 120.0 +rank5 [2026-10-27 20:47:43,818] INFO : Step 80000/100000 rank5 | Train Loss: 15.3065 | Train Loss Unweighted: 4.8564 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4208 | Regression_cos_phi: 0.0455 | Regression_energy: 0.6415 | Regression_eta: 0.0773 | Regression_pt: 0.6092 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6398 | physical_nopu: 4.4406 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2960 | Classification_binary: 3.5026 | Regression_cos_phi: 1.8251 | Regression_energy: 4.7275 | Regression_eta: 0.7889 | Regression_pt: 6.8359 | Regression_sin_phi: 1.4523 | LR: 7.44e-05 | DataLoad Time: 0.0086s | Model Forward Time: 3.0638s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0010 | max_forward: 0.0840 | max_loss: 0.0039 | max_backward: 0.0916 | max_optimizer_step: 0.2630 | max_tensorboard_logging: 1.7339 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5238.8 | uss_MB: 4942.0 | child_rss_MB: 7231.1 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 141.7 +rank5 [2026-10-27 20:47:44,748] INFO : Running validation on rank5 +rank5 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 20:49:38,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 20:49:38,981] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 20:50:05,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 20:50:05,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 20:50:05,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 20:50:06,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 20:50:06,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 20:50:07,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 20:50:07,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 20:50:08,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 20:50:18,310] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 20:50:18,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 20:50:18,745] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 20:50:33,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 20:50:34,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 20:50:34,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 20:50:35,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 20:50:35,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 20:50:36,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 20:50:36,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 20:50:37,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 20:50:47,259] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 20:50:47,267] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 20:50:47,678] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 20:51:02,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 20:51:03,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 20:51:03,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 20:51:04,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 20:51:04,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 20:51:04,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 20:51:05,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 20:51:05,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 20:51:15,807] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 20:54:47,405] INFO : Step 81000/100000 rank5 | Train Loss: 10.7072 | Train Loss Unweighted: 3.6241 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.9080 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2710 | Regression_eta: 0.1078 | Regression_pt: 0.2269 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6413 | physical_nopu: 4.4395 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3505 | Classification_binary: 2.9220 | Regression_cos_phi: 1.7703 | Regression_energy: 1.3118 | Regression_eta: 1.2322 | Regression_pt: 1.5989 | Regression_sin_phi: 1.6604 | LR: 7.11e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.4725s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0033 | max_forward: 0.1984 | max_loss: 0.0038 | max_backward: 0.3244 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 1.8991 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 7314.5 | uss_MB: 7027.3 | child_rss_MB: 7215.1 | allocated_MB: 408.0 | reserved_MB: 11650.0 | inactive_split_MB: 261.6 +rank5 [2026-10-27 20:58:04,710] INFO : Step 82000/100000 rank5 | Train Loss: 12.9094 | Train Loss Unweighted: 4.6043 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5247 | Regression_cos_phi: 0.0417 | Regression_energy: 0.4765 | Regression_eta: 0.0603 | Regression_pt: 0.4449 | Regression_sin_phi: 0.0402 | Group Train Loss Unweighted (window mean): physical_pu: 3.6409 | physical_nopu: 4.4420 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4154 | Classification_binary: 3.5656 | Regression_cos_phi: 1.5826 | Regression_energy: 1.9840 | Regression_eta: 0.6168 | Regression_pt: 2.6530 | Regression_sin_phi: 1.5168 | LR: 6.80e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.1712s | Phase Times: max_pre_step_barrier: 0.0156 | max_batch_to_device: 0.0008 | max_forward: 0.1472 | max_loss: 0.0038 | max_backward: 0.1553 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 1.8498 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5342.3 | uss_MB: 5059.9 | child_rss_MB: 7159.0 | allocated_MB: 393.8 | reserved_MB: 11650.0 | inactive_split_MB: 92.9 +rank5 [2026-10-27 21:01:20,554] INFO : Step 83000/100000 rank5 | Train Loss: 11.2556 | Train Loss Unweighted: 4.2618 | Train Loss Components: Classification: 0.0094 | Classification_binary: 3.5167 | Regression_cos_phi: 0.0501 | Regression_energy: 0.3127 | Regression_eta: 0.0669 | Regression_pt: 0.2586 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6408 | physical_nopu: 4.4397 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2134 | Classification_binary: 3.5005 | Regression_cos_phi: 1.7076 | Regression_energy: 1.5890 | Regression_eta: 0.7719 | Regression_pt: 1.9006 | Regression_sin_phi: 1.7196 | LR: 6.51e-05 | DataLoad Time: 0.0013s | Model Forward Time: 2.5758s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0008 | max_forward: 0.1683 | max_loss: 0.0037 | max_backward: 0.1023 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.3519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 5336.3 | uss_MB: 5061.6 | child_rss_MB: 7095.7 | allocated_MB: 393.2 | reserved_MB: 11650.0 | inactive_split_MB: 95.2 +rank5 [2026-10-27 21:04:33,664] INFO : Step 84000/100000 rank5 | Train Loss: 11.5964 | Train Loss Unweighted: 4.2611 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4346 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3486 | Regression_eta: 0.0638 | Regression_pt: 0.3055 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6366 | physical_nopu: 4.4419 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3504 | Classification_binary: 3.5182 | Regression_cos_phi: 1.4414 | Regression_energy: 1.3582 | Regression_eta: 0.6024 | Regression_pt: 1.7712 | Regression_sin_phi: 1.4489 | LR: 6.23e-05 | DataLoad Time: 0.0016s | Model Forward Time: 4.9341s | Phase Times: max_pre_step_barrier: 0.2933 | max_batch_to_device: 0.0008 | max_forward: 0.1129 | max_loss: 0.0110 | max_backward: 0.1312 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 3.4426 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5330.3 | uss_MB: 5063.1 | child_rss_MB: 7029.9 | allocated_MB: 393.9 | reserved_MB: 11650.0 | inactive_split_MB: 95.6 +rank5 [2026-10-27 21:07:28,408] INFO : Step 85000/100000 rank5 | Train Loss: 14.1090 | Train Loss Unweighted: 4.6641 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.3775 | Regression_cos_phi: 0.0425 | Regression_energy: 0.5754 | Regression_eta: 0.0615 | Regression_pt: 0.5479 | Regression_sin_phi: 0.0420 | Group Train Loss Unweighted (window mean): physical_pu: 3.6325 | physical_nopu: 4.4336 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3919 | Classification_binary: 3.4752 | Regression_cos_phi: 1.0978 | Regression_energy: 2.3453 | Regression_eta: 0.7519 | Regression_pt: 3.3313 | Regression_sin_phi: 1.4815 | LR: 5.96e-05 | DataLoad Time: 0.0021s | Model Forward Time: 2.5390s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0496 | max_backward: 0.1488 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 1.3583 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 5323.1 | uss_MB: 5063.4 | child_rss_MB: 7037.3 | allocated_MB: 394.0 | reserved_MB: 11698.0 | inactive_split_MB: 93.1 +rank5 [2026-10-27 21:10:26,799] INFO : Step 86000/100000 rank5 | Train Loss: 11.7921 | Train Loss Unweighted: 4.3412 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.5823 | Regression_cos_phi: 0.0537 | Regression_energy: 0.3118 | Regression_eta: 0.0750 | Regression_pt: 0.2521 | Regression_sin_phi: 0.0550 | Group Train Loss Unweighted (window mean): physical_pu: 3.6404 | physical_nopu: 4.4337 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2310 | Classification_binary: 3.5171 | Regression_cos_phi: 2.2986 | Regression_energy: 1.3647 | Regression_eta: 0.7122 | Regression_pt: 1.7176 | Regression_sin_phi: 1.9309 | LR: 5.71e-05 | DataLoad Time: 0.0023s | Model Forward Time: 2.4338s | Phase Times: max_pre_step_barrier: 0.0233 | max_batch_to_device: 0.0898 | max_forward: 0.1824 | max_loss: 0.0038 | max_backward: 0.0938 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.9995 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1323 | Memory: rss_MB: 5319.1 | uss_MB: 4616.7 | child_rss_MB: 7006.7 | allocated_MB: 393.2 | reserved_MB: 11704.0 | inactive_split_MB: 96.0 +rank5 [2026-10-27 21:13:31,178] INFO : Step 87000/100000 rank5 | Train Loss: 12.0054 | Train Loss Unweighted: 4.3232 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.3964 | Regression_cos_phi: 0.0444 | Regression_energy: 0.4046 | Regression_eta: 0.0620 | Regression_pt: 0.3567 | Regression_sin_phi: 0.0418 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3809 | Classification_binary: 3.4079 | Regression_cos_phi: 1.5306 | Regression_energy: 1.6436 | Regression_eta: 0.6966 | Regression_pt: 2.0500 | Regression_sin_phi: 1.3980 | LR: 5.48e-05 | DataLoad Time: 0.0018s | Model Forward Time: 2.3325s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.1290 | max_loss: 0.0039 | max_backward: 0.0975 | max_optimizer_step: 0.1784 | max_tensorboard_logging: 1.1620 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5310.9 | uss_MB: 5064.7 | child_rss_MB: 7011.5 | allocated_MB: 394.7 | reserved_MB: 11704.0 | inactive_split_MB: 94.3 +rank5 [2026-10-27 21:16:33,434] INFO : Step 88000/100000 rank5 | Train Loss: 14.2663 | Train Loss Unweighted: 4.6673 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.4533 | Regression_cos_phi: 0.0523 | Regression_energy: 0.5342 | Regression_eta: 0.0666 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0508 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3039 | Classification_binary: 3.5388 | Regression_cos_phi: 1.2233 | Regression_energy: 2.0447 | Regression_eta: 0.5553 | Regression_pt: 2.7512 | Regression_sin_phi: 1.4841 | LR: 5.26e-05 | DataLoad Time: 0.0008s | Model Forward Time: 2.3232s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0008 | max_forward: 0.0997 | max_loss: 0.0039 | max_backward: 0.1038 | max_optimizer_step: 0.2517 | max_tensorboard_logging: 1.0175 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5309.9 | uss_MB: 5065.2 | child_rss_MB: 7012.4 | allocated_MB: 393.7 | reserved_MB: 11704.0 | inactive_split_MB: 103.2 +rank5 [2026-10-27 21:19:30,808] INFO : Step 89000/100000 rank5 | Train Loss: 11.9885 | Train Loss Unweighted: 4.3122 | Train Loss Components: Classification: 0.0138 | Classification_binary: 3.4837 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3438 | Regression_eta: 0.0687 | Regression_pt: 0.2984 | Regression_sin_phi: 0.0554 | Group Train Loss Unweighted (window mean): physical_pu: 3.6329 | physical_nopu: 4.4374 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2727 | Classification_binary: 3.3409 | Regression_cos_phi: 1.4464 | Regression_energy: 1.6064 | Regression_eta: 0.7045 | Regression_pt: 1.9638 | Regression_sin_phi: 1.6376 | LR: 5.06e-05 | DataLoad Time: 0.0018s | Model Forward Time: 2.2337s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0976 | max_loss: 0.0053 | max_backward: 0.1242 | max_optimizer_step: 0.1785 | max_tensorboard_logging: 1.0326 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 5307.5 | uss_MB: 5066.4 | child_rss_MB: 7006.7 | allocated_MB: 395.0 | reserved_MB: 13422.0 | inactive_split_MB: 109.9 +rank5 [2026-10-27 21:22:22,522] INFO : Step 90000/100000 rank5 | Train Loss: 10.5888 | Train Loss Unweighted: 3.6256 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9282 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2656 | Regression_eta: 0.1044 | Regression_pt: 0.2178 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6354 | physical_nopu: 4.4354 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3389 | Classification_binary: 2.9427 | Regression_cos_phi: 1.6812 | Regression_energy: 1.2879 | Regression_eta: 1.1284 | Regression_pt: 1.5701 | Regression_sin_phi: 1.5758 | LR: 4.88e-05 | DataLoad Time: 0.0021s | Model Forward Time: 4.1185s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0033 | max_forward: 0.1953 | max_loss: 0.0645 | max_backward: 0.8258 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.9975 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5307.5 | uss_MB: 5074.7 | child_rss_MB: 7021.2 | allocated_MB: 404.6 | reserved_MB: 13422.0 | inactive_split_MB: 208.4 +rank5 [2026-10-27 21:22:23,511] INFO : Running validation on rank5 +rank5 [2026-10-27 21:22:23,511] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 21:22:23,511] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 21:22:23,511] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 21:22:23,511] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 21:24:23,625] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 21:24:24,118] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 21:24:48,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 21:24:49,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 21:24:49,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 21:24:50,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 21:24:50,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 21:24:51,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 21:24:51,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 21:24:52,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 21:25:02,195] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 21:25:02,472] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 21:25:02,871] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 21:25:18,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 21:25:18,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 21:25:19,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 21:25:20,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 21:25:20,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 21:25:21,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 21:25:21,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 21:25:22,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 21:25:32,087] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 21:25:32,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 21:25:32,493] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 21:25:47,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 21:25:47,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 21:25:48,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 21:25:48,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 21:25:49,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 21:25:49,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 21:25:50,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 21:25:50,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 21:26:00,757] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 21:29:48,334] INFO : Step 91000/100000 rank5 | Train Loss: 11.9808 | Train Loss Unweighted: 4.3520 | Train Loss Components: Classification: 0.0128 | Classification_binary: 3.5260 | Regression_cos_phi: 0.0522 | Regression_energy: 0.3480 | Regression_eta: 0.0723 | Regression_pt: 0.2901 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4346 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2630 | Classification_binary: 3.4025 | Regression_cos_phi: 1.8305 | Regression_energy: 1.3596 | Regression_eta: 0.7104 | Regression_pt: 1.6480 | Regression_sin_phi: 1.4802 | LR: 4.71e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.6012s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1743 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.3267 | max_tensorboard_logging: 2.0733 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0716 | Memory: rss_MB: 7308.2 | uss_MB: 7066.1 | child_rss_MB: 7044.9 | allocated_MB: 393.7 | reserved_MB: 11392.0 | inactive_split_MB: 89.1 +rank5 [2026-10-27 21:33:08,656] INFO : Step 92000/100000 rank5 | Train Loss: 14.4974 | Train Loss Unweighted: 4.7629 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4460 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5917 | Regression_eta: 0.0739 | Regression_pt: 0.5467 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6297 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3242 | Classification_binary: 3.5020 | Regression_cos_phi: 1.7004 | Regression_energy: 1.9965 | Regression_eta: 0.8551 | Regression_pt: 2.5997 | Regression_sin_phi: 1.2947 | LR: 4.57e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.6196s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1620 | max_loss: 0.0037 | max_backward: 0.2478 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 1.1676 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0595 | Memory: rss_MB: 7302.2 | uss_MB: 6481.6 | child_rss_MB: 6997.3 | allocated_MB: 394.0 | reserved_MB: 11392.0 | inactive_split_MB: 90.6 +rank5 [2026-10-27 21:36:11,295] INFO : Step 93000/100000 rank5 | Train Loss: 13.1041 | Train Loss Unweighted: 4.5189 | Train Loss Components: Classification: 0.0119 | Classification_binary: 3.5265 | Regression_cos_phi: 0.0575 | Regression_energy: 0.4342 | Regression_eta: 0.0649 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6346 | physical_nopu: 4.4327 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2100 | Classification_binary: 3.5966 | Regression_cos_phi: 1.7999 | Regression_energy: 1.6819 | Regression_eta: 0.8352 | Regression_pt: 1.9080 | Regression_sin_phi: 1.5045 | LR: 4.43e-05 | DataLoad Time: 0.0021s | Model Forward Time: 2.4221s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1013 | max_loss: 0.1069 | max_backward: 0.2035 | max_optimizer_step: 0.1792 | max_tensorboard_logging: 1.1596 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 5379.1 | uss_MB: 5141.2 | child_rss_MB: 6951.6 | allocated_MB: 393.6 | reserved_MB: 13294.0 | inactive_split_MB: 89.2 +rank5 [2026-10-27 21:39:11,302] INFO : Step 94000/100000 rank5 | Train Loss: 11.5177 | Train Loss Unweighted: 4.2756 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4441 | Regression_cos_phi: 0.0413 | Regression_energy: 0.3503 | Regression_eta: 0.0686 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6308 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3719 | Classification_binary: 3.6494 | Regression_cos_phi: 1.7441 | Regression_energy: 1.6151 | Regression_eta: 0.8798 | Regression_pt: 2.0680 | Regression_sin_phi: 1.8699 | LR: 4.32e-05 | DataLoad Time: 0.0016s | Model Forward Time: 6.3169s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0191 | max_forward: 0.1721 | max_loss: 0.0188 | max_backward: 0.2028 | max_optimizer_step: 0.2895 | max_tensorboard_logging: 4.7123 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 5372.1 | uss_MB: 5157.2 | child_rss_MB: 6925.7 | allocated_MB: 394.1 | reserved_MB: 13294.0 | inactive_split_MB: 89.1 +rank5 [2026-10-27 21:42:20,694] INFO : Step 95000/100000 rank5 | Train Loss: 14.1207 | Train Loss Unweighted: 4.6477 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3920 | Regression_cos_phi: 0.0441 | Regression_energy: 0.5776 | Regression_eta: 0.0602 | Regression_pt: 0.5087 | Regression_sin_phi: 0.0499 | Group Train Loss Unweighted (window mean): physical_pu: 3.6267 | physical_nopu: 4.4321 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3555 | Classification_binary: 3.5281 | Regression_cos_phi: 1.6527 | Regression_energy: 2.1608 | Regression_eta: 0.4944 | Regression_pt: 2.8690 | Regression_sin_phi: 1.5292 | LR: 4.22e-05 | DataLoad Time: 0.0011s | Model Forward Time: 2.5043s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.0941 | max_loss: 0.0036 | max_backward: 0.1734 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 1.2811 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5361.0 | uss_MB: 5157.8 | child_rss_MB: 6928.3 | allocated_MB: 396.0 | reserved_MB: 14294.0 | inactive_split_MB: 91.1 +rank5 [2026-10-27 21:45:16,251] INFO : Step 96000/100000 rank5 | Train Loss: 11.5036 | Train Loss Unweighted: 4.2276 | Train Loss Components: Classification: 0.0097 | Classification_binary: 3.4897 | Regression_cos_phi: 0.0538 | Regression_energy: 0.2982 | Regression_eta: 0.0802 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6278 | physical_nopu: 4.4329 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2502 | Classification_binary: 3.3215 | Regression_cos_phi: 1.5205 | Regression_energy: 1.3893 | Regression_eta: 0.7782 | Regression_pt: 1.5887 | Regression_sin_phi: 1.6500 | LR: 4.14e-05 | DataLoad Time: 0.0024s | Model Forward Time: 2.3867s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0007 | max_forward: 0.0991 | max_loss: 0.0037 | max_backward: 0.1000 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5357.0 | uss_MB: 5158.4 | child_rss_MB: 6899.7 | allocated_MB: 393.0 | reserved_MB: 14294.0 | inactive_split_MB: 95.3 +rank5 [2026-10-27 21:48:34,573] INFO : Step 97000/100000 rank5 | Train Loss: 11.8793 | Train Loss Unweighted: 4.3044 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.4420 | Regression_cos_phi: 0.0454 | Regression_energy: 0.3663 | Regression_eta: 0.0711 | Regression_pt: 0.3156 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.6277 | physical_nopu: 4.4255 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3076 | Classification_binary: 3.4731 | Regression_cos_phi: 1.8532 | Regression_energy: 2.0851 | Regression_eta: 0.5991 | Regression_pt: 2.5793 | Regression_sin_phi: 1.6874 | LR: 4.08e-05 | DataLoad Time: 0.0007s | Model Forward Time: 2.5926s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1192 | max_loss: 0.0048 | max_backward: 0.1852 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 5356.0 | uss_MB: 5157.9 | child_rss_MB: 6903.2 | allocated_MB: 394.7 | reserved_MB: 14294.0 | inactive_split_MB: 98.1 +rank5 [2026-10-27 21:51:32,851] INFO : Step 98000/100000 rank5 | Train Loss: 12.9367 | Train Loss Unweighted: 4.4840 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0441 | Regression_energy: 0.4811 | Regression_eta: 0.0611 | Regression_pt: 0.4378 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3484 | Classification_binary: 3.2512 | Regression_cos_phi: 1.2011 | Regression_energy: 2.6866 | Regression_eta: 0.5545 | Regression_pt: 3.5903 | Regression_sin_phi: 1.1030 | LR: 4.04e-05 | DataLoad Time: 0.0012s | Model Forward Time: 2.8950s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1318 | max_loss: 0.0038 | max_backward: 0.1127 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.6868 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 5362.0 | uss_MB: 5160.3 | child_rss_MB: 6899.7 | allocated_MB: 394.5 | reserved_MB: 14294.0 | inactive_split_MB: 82.0 +rank5 [2026-10-27 21:54:35,739] INFO : Step 99000/100000 rank5 | Train Loss: 11.6408 | Train Loss Unweighted: 4.3204 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.5557 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3117 | Regression_eta: 0.0641 | Regression_pt: 0.2727 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.6292 | physical_nopu: 4.4284 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2267 | Classification_binary: 3.4783 | Regression_cos_phi: 2.1157 | Regression_energy: 1.3163 | Regression_eta: 0.5744 | Regression_pt: 1.7344 | Regression_sin_phi: 1.6584 | LR: 4.01e-05 | DataLoad Time: 0.0082s | Model Forward Time: 3.0807s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1731 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.8331 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 5358.0 | uss_MB: 5160.5 | child_rss_MB: 6886.0 | allocated_MB: 394.3 | reserved_MB: 14294.0 | inactive_split_MB: 113.7 +rank5 [2026-10-27 21:57:56,508] INFO : Step 100000/100000 rank5 | Train Loss: 10.8909 | Train Loss Unweighted: 3.6545 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9133 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2833 | Regression_eta: 0.1041 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6336 | physical_nopu: 4.4311 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3859 | Classification_binary: 2.9201 | Regression_cos_phi: 1.6776 | Regression_energy: 1.3781 | Regression_eta: 1.1791 | Regression_pt: 1.7011 | Regression_sin_phi: 1.6709 | LR: 4.00e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.7457s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0030 | max_forward: 0.2088 | max_loss: 0.0393 | max_backward: 0.3031 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.9276 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 5350.9 | uss_MB: 5163.3 | child_rss_MB: 6834.2 | allocated_MB: 405.7 | reserved_MB: 14294.0 | inactive_split_MB: 165.9 +rank5 [2026-10-27 21:57:57,460] INFO : Running validation on rank5 +rank5 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 21:59:51,634] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 21:59:52,048] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 22:00:18,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 22:00:19,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 22:00:19,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 22:00:20,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 22:00:20,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 22:00:21,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 22:00:21,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 22:00:21,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 22:00:31,998] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 22:00:32,062] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 22:00:32,464] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 22:00:47,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 22:00:47,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 22:00:48,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 22:00:48,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 22:00:49,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 22:00:49,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 22:00:50,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 22:00:50,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 22:01:00,830] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 22:01:01,522] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 22:01:01,928] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 22:01:16,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 22:01:17,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 22:01:17,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 22:01:18,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 22:01:18,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 22:01:19,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 22:01:19,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 22:01:19,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 22:01:29,866] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 22:01:30,998] INFO : Training completed. Total time on device 5: 350.234min diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.6 b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.6 new file mode 100644 index 0000000000000000000000000000000000000000..7498714ad33f686608635d691117e18a8f6d5f57 --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.6 @@ -0,0 +1,851 @@ +rank6 [2026-10-27 16:10:10,443] INFO : Initializing model with seed=12345; process seed=12351 +rank6 [2026-10-27 16:10:10,443] INFO : configured dtype=torch.bfloat16 for autocast +rank6 [2026-10-27 16:10:37,169] INFO : Instantiating model +rank6 [2026-10-27 16:10:37,169] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank6 [2026-10-27 16:10:37,169] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank6 [2026-10-27 16:10:37,201] INFO : Input encoding initialization took 0.03s +rank6 [2026-10-27 16:10:37,201] INFO : nn0 parameters: 2130944 +rank6 [2026-10-27 16:10:37,201] INFO : Initializing shared backbone layers, num_convs=3 +rank6 [2026-10-27 16:10:37,201] INFO : Initializing backbone layer 0 +rank6 [2026-10-27 16:10:37,201] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-27 16:10:37,212] INFO : Initializing backbone layer 1 +rank6 [2026-10-27 16:10:37,212] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-27 16:10:37,223] INFO : Initializing backbone layer 2 +rank6 [2026-10-27 16:10:37,223] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-27 16:10:37,233] INFO : Backbone initialization took 0.03s +rank6 [2026-10-27 16:10:37,234] INFO : backbone parameters: 4733952 +rank6 [2026-10-27 16:10:37,234] INFO : Initializing output DNNs +rank6 [2026-10-27 16:10:37,268] INFO : Output DNNs initialization took 0.03s +rank6 [2026-10-27 16:10:37,268] INFO : backbone_mode=BackboneMode.SHARED +rank6 [2026-10-27 16:10:37,268] INFO : jagged_attention enabled=True +rank6 [2026-10-27 16:10:37,268] INFO : flash_attn_varlen enabled=True +rank6 [2026-10-27 16:10:37,268] INFO : task_queries enabled=False +rank6 [2026-10-27 16:10:37,268] INFO : classification_norm parameters: 1024 +rank6 [2026-10-27 16:10:37,268] INFO : regression_norm parameters: 1024 +rank6 [2026-10-27 16:10:37,268] INFO : classification_query parameters: 0 +rank6 [2026-10-27 16:10:37,268] INFO : regression_query parameters: 0 +rank6 [2026-10-27 16:10:37,269] INFO : classification_readout parameters: 0 +rank6 [2026-10-27 16:10:37,269] INFO : regression_readout parameters: 0 +rank6 [2026-10-27 16:10:37,269] INFO : output_mode=elementwise +rank6 [2026-10-27 16:10:37,269] INFO : set_decoder parameters: 0 +rank6 [2026-10-27 16:10:37,269] INFO : nn_binary_particle parameters: 264706 +rank6 [2026-10-27 16:10:37,269] INFO : nn_pid parameters: 268297 +rank6 [2026-10-27 16:10:37,269] INFO : nn_pt parameters: 2113544 +rank6 [2026-10-27 16:10:37,269] INFO : nn_eta parameters: 264706 +rank6 [2026-10-27 16:10:37,269] INFO : nn_sin_phi parameters: 264706 +rank6 [2026-10-27 16:10:37,269] INFO : nn_cos_phi parameters: 264706 +rank6 [2026-10-27 16:10:37,269] INFO : nn_energy parameters: 2113544 +rank6 [2026-10-27 16:10:37,270] INFO : Total MLPF parameters: 12421153 +rank6 [2026-10-27 16:10:37,270] INFO : MLPF __init__ done +rank6 [2026-10-27 16:10:37,270] INFO : Instantiated model +rank6 [2026-10-27 16:10:37,270] INFO : Moving model to device rank=6 +rank6 [2026-10-27 16:10:37,299] INFO : Moved model to device rank=6 +rank6 [2026-10-27 16:10:37,301] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank6 [2026-10-27 16:10:37,301] INFO : Configured model for SyncBatchNorm rank=6 +rank6 [2026-10-27 16:10:37,493] INFO : Configured model for DistributedDataParallel rank=6 +rank6 [2026-10-27 16:10:37,593] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +167 module.nn_energy.nn.7.0.bias ... 0 +168 module.nn_energy.nn.7.2.weight ... 0 +169 module.nn_energy.nn.7.2.bias ... 0 +170 module.nn_energy.nn.7.4.weight ... 0 +171 module.nn_energy.nn.7.4.bias ... 0 + +[172 rows x 3 columns] +rank6 [2026-10-27 16:10:37,593] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=512, out_features=512, bias=True) + ) + (norm0): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=512, out_features=512, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank6 [2026-10-27 16:10:37,594] INFO : Trainable parameters: 12421153 +rank6 [2026-10-27 16:10:37,594] INFO : Non-trainable parameters: 0 +rank6 [2026-10-27 16:10:37,594] INFO : Total parameters: 12421153 +rank6 [2026-10-27 16:10:37,596] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.mha.in_proj_weight 786432 0 + module.backbone.0.mha.in_proj_bias 1536 0 +module.backbone.0.mha.out_proj.weight 262144 0 + module.backbone.0.mha.out_proj.bias 512 0 + module.backbone.0.norm0.weight 512 0 + module.backbone.0.norm0.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm1.bias 512 0 + module.backbone.0.seq.0.weight 262144 0 + module.backbone.0.seq.0.bias 512 0 + module.backbone.0.seq.2.weight 262144 0 + module.backbone.0.seq.2.bias 512 0 + module.backbone.1.mha.in_proj_weight 786432 0 + module.backbone.1.mha.in_proj_bias 1536 0 +module.backbone.1.mha.out_proj.weight 262144 0 + module.backbone.1.mha.out_proj.bias 512 0 + module.backbone.1.norm0.weight 512 0 + module.backbone.1.norm0.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm1.bias 512 0 + module.backbone.1.seq.0.weight 262144 0 + module.backbone.1.seq.0.bias 512 0 + module.backbone.1.seq.2.weight 262144 0 + module.backbone.1.seq.2.bias 512 0 + module.backbone.2.mha.in_proj_weight 786432 0 + module.backbone.2.mha.in_proj_bias 1536 0 +module.backbone.2.mha.out_proj.weight 262144 0 + module.backbone.2.mha.out_proj.bias 512 0 + module.backbone.2.norm0.weight 512 0 + module.backbone.2.norm0.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm1.bias 512 0 + module.backbone.2.seq.0.weight 262144 0 + module.backbone.2.seq.0.bias 512 0 + module.backbone.2.seq.2.weight 262144 0 + module.backbone.2.seq.2.bias 512 0 + module.classification_norm.weight 512 0 + module.classification_norm.bias 512 0 + module.regression_norm.weight 512 0 + module.regression_norm.bias 512 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank6 [2026-10-27 16:10:37,596] INFO : Getting dataloaders +rank6 [2026-10-27 16:10:37,596] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:43,312] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:44,041] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:44,856] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank6 [2026-10-27 16:10:44,856] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:45,822] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:46,988] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:47,782] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank6 [2026-10-27 16:10:47,782] INFO : Creating WeightedMixIterator with 2 data loaders. +rank6 [2026-10-27 16:10:47,782] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:47,934] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:48,087] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:48,255] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank6 [2026-10-27 16:10:48,255] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:48,427] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:48,849] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:48,992] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank6 [2026-10-27 16:10:48,992] INFO : Creating InterleavedIterator with 2 data loaders. +rank6 [2026-10-27 16:11:17,096] INFO : Got dataloaders +rank6 [2026-10-27 16:11:17,096] INFO : loader split=train rank=6 len=232058 +rank6 [2026-10-27 16:11:17,096] INFO : loader split=valid rank=6 len=48 +rank6 [2026-10-27 16:11:17,096] INFO : Starting training from step 1 to 100000 on rank 6 of 8 +rank6 [2026-10-27 16:11:17,096] INFO : Creating train_iterator +rank6 [2026-10-27 16:11:17,097] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank6 [2026-10-27 16:11:31,019] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank6 [2026-10-27 16:11:46,395] INFO : Created train_iterator +rank6 [2026-10-27 16:12:23,646] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.123548031449318, 'Classification': 0.2298217851272784, 'Regression_pt': 0.7194055180624127, 'Regression_eta': 0.4619348940253258, 'Regression_sin_phi': 0.15005176125559955, 'Regression_cos_phi': 0.14165307599119842, 'Regression_energy': 1.0569261094555258}, weights={'Classification_binary': 1.0, 'Classification': 22.293569564819336, 'Regression_pt': 7.12191915512085, 'Regression_eta': 11.091493606567383, 'Regression_sin_phi': 34.14520263671875, 'Regression_cos_phi': 36.16969299316406, 'Regression_energy': 4.847593307495117} +rank6 [2026-10-27 16:14:48,547] INFO : Step 1000/100000 rank6 | Train Loss: 15.9738 | Train Loss Unweighted: 5.5021 | Train Loss Components: Classification: 0.0191 | Classification_binary: 4.1436 | Regression_cos_phi: 0.0516 | Regression_energy: 0.6035 | Regression_eta: 0.0819 | Regression_pt: 0.5500 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 5.2846 | physical_nopu: 6.1663 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.5038 | Classification_binary: 4.0742 | Regression_cos_phi: 1.7451 | Regression_energy: 2.4592 | Regression_eta: 0.6674 | Regression_pt: 3.2791 | Regression_sin_phi: 1.9342 | LR: 4.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9580s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0013 | max_forward: 0.0194 | max_loss: 0.0039 | max_backward: 0.0308 | max_optimizer_step: 0.1391 | max_tensorboard_logging: 1.0816 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0030 | Memory: rss_MB: 4094.8 | uss_MB: 2885.6 | child_rss_MB: 10815.8 | allocated_MB: 395.5 | reserved_MB: 11198.0 | inactive_split_MB: 104.9 +rank6 [2026-10-27 16:17:31,218] INFO : Step 2000/100000 rank6 | Train Loss: 17.2752 | Train Loss Unweighted: 5.6831 | Train Loss Components: Classification: 0.0157 | Classification_binary: 4.0887 | Regression_cos_phi: 0.0506 | Regression_energy: 0.7236 | Regression_eta: 0.0696 | Regression_pt: 0.6798 | Regression_sin_phi: 0.0552 | Group Train Loss Unweighted (window mean): physical_pu: 4.4021 | physical_nopu: 5.4797 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4633 | Classification_binary: 3.9608 | Regression_cos_phi: 2.0502 | Regression_energy: 3.9318 | Regression_eta: 0.9281 | Regression_pt: 5.6299 | Regression_sin_phi: 2.7279 | LR: 4.00e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.7795s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0250 | max_loss: 0.0036 | max_backward: 0.0657 | max_optimizer_step: 0.1397 | max_tensorboard_logging: 0.8949 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4097.7 | uss_MB: 2895.6 | child_rss_MB: 10834.7 | allocated_MB: 394.2 | reserved_MB: 11198.0 | inactive_split_MB: 90.1 +rank6 [2026-10-27 16:20:19,985] INFO : Step 3000/100000 rank6 | Train Loss: 14.0295 | Train Loss Unweighted: 5.1258 | Train Loss Components: Classification: 0.0148 | Classification_binary: 4.0405 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4749 | Regression_eta: 0.0746 | Regression_pt: 0.4205 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 4.1913 | physical_nopu: 5.3261 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3991 | Classification_binary: 4.1547 | Regression_cos_phi: 2.3407 | Regression_energy: 2.1524 | Regression_eta: 0.9655 | Regression_pt: 2.8381 | Regression_sin_phi: 1.9051 | LR: 3.99e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.4926s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1067 | max_loss: 0.0036 | max_backward: 0.0365 | max_optimizer_step: 0.1415 | max_tensorboard_logging: 1.1098 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0093 | Memory: rss_MB: 4102.6 | uss_MB: 2907.4 | child_rss_MB: 10837.4 | allocated_MB: 395.4 | reserved_MB: 11784.0 | inactive_split_MB: 88.5 +rank6 [2026-10-27 16:23:08,050] INFO : Step 4000/100000 rank6 | Train Loss: 11.7244 | Train Loss Unweighted: 3.9925 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.1718 | Regression_cos_phi: 0.0487 | Regression_energy: 0.3212 | Regression_eta: 0.1125 | Regression_pt: 0.2729 | Regression_sin_phi: 0.0492 | Group Train Loss Unweighted (window mean): physical_pu: 4.0838 | physical_nopu: 5.2370 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3681 | Classification_binary: 3.1460 | Regression_cos_phi: 1.8128 | Regression_energy: 1.5451 | Regression_eta: 1.3283 | Regression_pt: 1.9261 | Regression_sin_phi: 1.7578 | LR: 3.99e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.2687s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0037 | max_forward: 0.0726 | max_loss: 0.0066 | max_backward: 0.1670 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4108.6 | uss_MB: 2925.8 | child_rss_MB: 10838.2 | allocated_MB: 406.4 | reserved_MB: 11904.0 | inactive_split_MB: 179.6 +rank6 [2026-10-27 16:25:55,983] INFO : Step 5000/100000 rank6 | Train Loss: 13.1313 | Train Loss Unweighted: 4.9481 | Train Loss Components: Classification: 0.0138 | Classification_binary: 4.0144 | Regression_cos_phi: 0.0518 | Regression_energy: 0.3902 | Regression_eta: 0.0928 | Regression_pt: 0.3380 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 4.0057 | physical_nopu: 5.1582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3182 | Classification_binary: 4.0429 | Regression_cos_phi: 2.7912 | Regression_energy: 1.9610 | Regression_eta: 0.8719 | Regression_pt: 2.3912 | Regression_sin_phi: 1.6002 | LR: 3.98e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.3569s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0196 | max_loss: 0.0037 | max_backward: 0.0324 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 1.3896 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0068 | Memory: rss_MB: 4091.1 | uss_MB: 2927.1 | child_rss_MB: 10673.2 | allocated_MB: 393.9 | reserved_MB: 11904.0 | inactive_split_MB: 90.0 +rank6 [2026-10-27 16:28:58,362] INFO : Step 6000/100000 rank6 | Train Loss: 14.5226 | Train Loss Unweighted: 5.0639 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.8601 | Regression_cos_phi: 0.0525 | Regression_energy: 0.5359 | Regression_eta: 0.0694 | Regression_pt: 0.4825 | Regression_sin_phi: 0.0459 | Group Train Loss Unweighted (window mean): physical_pu: 3.9666 | physical_nopu: 5.0829 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3727 | Classification_binary: 3.8091 | Regression_cos_phi: 2.0647 | Regression_energy: 2.3927 | Regression_eta: 1.1117 | Regression_pt: 3.4000 | Regression_sin_phi: 1.4964 | LR: 3.97e-04 | DataLoad Time: 0.0015s | Model Forward Time: 3.5997s | Phase Times: max_pre_step_barrier: 0.0280 | max_batch_to_device: 0.0008 | max_forward: 0.0311 | max_loss: 0.0037 | max_backward: 0.0629 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 2.6604 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0119 | Memory: rss_MB: 4078.1 | uss_MB: 2934.4 | child_rss_MB: 10495.3 | allocated_MB: 394.6 | reserved_MB: 12524.0 | inactive_split_MB: 109.0 +rank6 [2026-10-27 16:31:51,870] INFO : Step 7000/100000 rank6 | Train Loss: 13.9772 | Train Loss Unweighted: 5.0398 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.9636 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4671 | Regression_eta: 0.0797 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0537 | Group Train Loss Unweighted (window mean): physical_pu: 3.9390 | physical_nopu: 5.0316 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2236 | Classification_binary: 4.1187 | Regression_cos_phi: 1.8015 | Regression_energy: 2.2774 | Regression_eta: 0.9644 | Regression_pt: 2.7724 | Regression_sin_phi: 1.9661 | LR: 3.96e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.1748s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0243 | max_loss: 0.0036 | max_backward: 0.0295 | max_optimizer_step: 0.2429 | max_tensorboard_logging: 2.1406 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 4070.6 | uss_MB: 2937.8 | child_rss_MB: 10370.1 | allocated_MB: 393.9 | reserved_MB: 12524.0 | inactive_split_MB: 107.5 +rank6 [2026-10-27 16:35:03,086] INFO : Step 8000/100000 rank6 | Train Loss: 13.6161 | Train Loss Unweighted: 4.9650 | Train Loss Components: Classification: 0.0187 | Classification_binary: 3.9275 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4519 | Regression_eta: 0.0615 | Regression_pt: 0.4056 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.9031 | physical_nopu: 4.9878 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4574 | Classification_binary: 3.7951 | Regression_cos_phi: 1.8596 | Regression_energy: 2.2365 | Regression_eta: 0.6916 | Regression_pt: 3.0724 | Regression_sin_phi: 1.4739 | LR: 3.94e-04 | DataLoad Time: 0.0015s | Model Forward Time: 2.9576s | Phase Times: max_pre_step_barrier: 0.0692 | max_batch_to_device: 0.0893 | max_forward: 0.1169 | max_loss: 0.0037 | max_backward: 0.0488 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 1.9012 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4069.6 | uss_MB: 2945.4 | child_rss_MB: 10376.1 | allocated_MB: 393.8 | reserved_MB: 12524.0 | inactive_split_MB: 105.1 +rank6 [2026-10-27 16:38:07,477] INFO : Step 9000/100000 rank6 | Train Loss: 15.1606 | Train Loss Unweighted: 5.0855 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.7493 | Regression_cos_phi: 0.0474 | Regression_energy: 0.5909 | Regression_eta: 0.0749 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.8927 | physical_nopu: 4.9579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3626 | Classification_binary: 3.7181 | Regression_cos_phi: 1.8991 | Regression_energy: 2.0402 | Regression_eta: 0.7344 | Regression_pt: 2.6912 | Regression_sin_phi: 1.8622 | LR: 3.93e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.8503s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0007 | max_forward: 0.0386 | max_loss: 0.0041 | max_backward: 0.0582 | max_optimizer_step: 0.1496 | max_tensorboard_logging: 1.9225 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 4056.6 | uss_MB: 2948.9 | child_rss_MB: 10378.8 | allocated_MB: 395.8 | reserved_MB: 12524.0 | inactive_split_MB: 89.5 +rank6 [2026-10-27 16:41:18,312] INFO : Step 10000/100000 rank6 | Train Loss: 12.2525 | Train Loss Unweighted: 4.6725 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8667 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3374 | Regression_eta: 0.0750 | Regression_pt: 0.2772 | Regression_sin_phi: 0.0525 | Group Train Loss Unweighted (window mean): physical_pu: 3.8762 | physical_nopu: 4.9262 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2098 | Classification_binary: 3.9210 | Regression_cos_phi: 2.1774 | Regression_energy: 1.4771 | Regression_eta: 1.0103 | Regression_pt: 1.5700 | Regression_sin_phi: 1.4631 | LR: 3.91e-04 | DataLoad Time: 0.0014s | Model Forward Time: 3.1914s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0277 | max_loss: 0.0035 | max_backward: 0.0304 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 2.2107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4024.3 | uss_MB: 2953.8 | child_rss_MB: 10382.4 | allocated_MB: 393.6 | reserved_MB: 12524.0 | inactive_split_MB: 89.8 +rank6 [2026-10-27 16:41:19,252] INFO : Running validation on rank6 +rank6 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 16:43:24,652] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:43:25,081] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 16:43:52,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 16:43:52,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 16:43:53,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 16:43:53,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 16:43:54,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 16:43:54,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 16:43:55,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 16:43:55,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 16:44:05,965] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-27 16:44:06,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:44:06,938] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 16:44:22,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 16:44:22,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 16:44:22,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 16:44:23,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 16:44:23,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 16:44:24,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 16:44:24,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 16:44:25,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 16:44:35,228] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 16:44:35,783] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:44:36,275] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 16:44:51,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 16:44:51,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 16:44:52,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 16:44:52,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 16:44:53,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 16:44:53,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 16:44:54,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 16:44:54,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 16:45:04,660] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 16:48:51,455] INFO : Step 11000/100000 rank6 | Train Loss: 13.5331 | Train Loss Unweighted: 4.8490 | Train Loss Components: Classification: 0.0178 | Classification_binary: 3.8332 | Regression_cos_phi: 0.0514 | Regression_energy: 0.4326 | Regression_eta: 0.0715 | Regression_pt: 0.3906 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.8629 | physical_nopu: 4.8860 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4077 | Classification_binary: 3.7651 | Regression_cos_phi: 1.3643 | Regression_energy: 2.6189 | Regression_eta: 0.7208 | Regression_pt: 3.7318 | Regression_sin_phi: 1.5858 | LR: 3.89e-04 | DataLoad Time: 0.0029s | Model Forward Time: 3.1300s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0010 | max_forward: 0.0938 | max_loss: 0.0036 | max_backward: 0.1077 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.9405 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6748.2 | uss_MB: 5693.6 | child_rss_MB: 10378.2 | allocated_MB: 394.1 | reserved_MB: 11016.0 | inactive_split_MB: 102.9 +rank6 [2026-10-27 16:51:46,972] INFO : Step 12000/100000 rank6 | Train Loss: 16.0880 | Train Loss Unweighted: 5.2955 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.7102 | Regression_cos_phi: 0.0409 | Regression_energy: 0.7300 | Regression_eta: 0.0621 | Regression_pt: 0.6963 | Regression_sin_phi: 0.0392 | Group Train Loss Unweighted (window mean): physical_pu: 3.8522 | physical_nopu: 4.8676 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3946 | Classification_binary: 3.8180 | Regression_cos_phi: 1.5051 | Regression_energy: 2.8630 | Regression_eta: 0.7070 | Regression_pt: 4.0202 | Regression_sin_phi: 1.2999 | LR: 3.87e-04 | DataLoad Time: 0.0008s | Model Forward Time: 2.9546s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1050 | max_forward: 0.2073 | max_loss: 0.0038 | max_backward: 0.1040 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 1.6163 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6734.2 | uss_MB: 5695.7 | child_rss_MB: 10317.1 | allocated_MB: 394.0 | reserved_MB: 11384.0 | inactive_split_MB: 88.0 +rank6 [2026-10-27 16:54:47,130] INFO : Step 13000/100000 rank6 | Train Loss: 12.2526 | Train Loss Unweighted: 4.6994 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.8775 | Regression_cos_phi: 0.0480 | Regression_energy: 0.3426 | Regression_eta: 0.0688 | Regression_pt: 0.2970 | Regression_sin_phi: 0.0539 | Group Train Loss Unweighted (window mean): physical_pu: 3.8404 | physical_nopu: 4.8415 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2698 | Classification_binary: 3.8409 | Regression_cos_phi: 1.8816 | Regression_energy: 1.3431 | Regression_eta: 0.6897 | Regression_pt: 1.6701 | Regression_sin_phi: 1.7518 | LR: 3.85e-04 | DataLoad Time: 0.0061s | Model Forward Time: 3.3044s | Phase Times: max_pre_step_barrier: 0.0143 | max_batch_to_device: 0.0010 | max_forward: 0.0740 | max_loss: 0.0036 | max_backward: 0.0639 | max_optimizer_step: 0.2642 | max_tensorboard_logging: 2.1016 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 5345.9 | uss_MB: 4318.0 | child_rss_MB: 10308.5 | allocated_MB: 394.7 | reserved_MB: 12168.0 | inactive_split_MB: 111.5 +rank6 [2026-10-27 16:57:46,633] INFO : Step 14000/100000 rank6 | Train Loss: 11.1328 | Train Loss Unweighted: 3.8032 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.0595 | Regression_cos_phi: 0.0494 | Regression_energy: 0.2823 | Regression_eta: 0.1077 | Regression_pt: 0.2396 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.8281 | physical_nopu: 4.8178 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3500 | Classification_binary: 3.0828 | Regression_cos_phi: 1.7786 | Regression_energy: 1.4264 | Regression_eta: 1.2482 | Regression_pt: 1.7747 | Regression_sin_phi: 1.6783 | LR: 3.83e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.7658s | Phase Times: max_pre_step_barrier: 0.1064 | max_batch_to_device: 0.0033 | max_forward: 0.1130 | max_loss: 0.0036 | max_backward: 0.2788 | max_optimizer_step: 0.1663 | max_tensorboard_logging: 2.1758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1099 | Memory: rss_MB: 5335.9 | uss_MB: 4329.8 | child_rss_MB: 10259.4 | allocated_MB: 405.3 | reserved_MB: 12168.0 | inactive_split_MB: 126.3 +rank6 [2026-10-27 17:00:50,805] INFO : Step 15000/100000 rank6 | Train Loss: 12.4223 | Train Loss Unweighted: 4.6301 | Train Loss Components: Classification: 0.0132 | Classification_binary: 3.7975 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3484 | Regression_eta: 0.0765 | Regression_pt: 0.2883 | Regression_sin_phi: 0.0502 | Group Train Loss Unweighted (window mean): physical_pu: 3.8205 | physical_nopu: 4.8025 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2171 | Classification_binary: 3.8583 | Regression_cos_phi: 1.6522 | Regression_energy: 1.2886 | Regression_eta: 0.6336 | Regression_pt: 1.5598 | Regression_sin_phi: 1.6154 | LR: 3.80e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.2224s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0799 | max_loss: 0.0851 | max_backward: 0.1383 | max_optimizer_step: 0.1621 | max_tensorboard_logging: 2.1079 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5320.9 | uss_MB: 4324.4 | child_rss_MB: 10137.7 | allocated_MB: 394.1 | reserved_MB: 12168.0 | inactive_split_MB: 89.6 +rank6 [2026-10-27 17:04:12,219] INFO : Step 16000/100000 rank6 | Train Loss: 13.1559 | Train Loss Unweighted: 4.6950 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.6917 | Regression_cos_phi: 0.0473 | Regression_energy: 0.4330 | Regression_eta: 0.0771 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.8147 | physical_nopu: 4.7795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4155 | Classification_binary: 3.6960 | Regression_cos_phi: 1.9759 | Regression_energy: 2.1119 | Regression_eta: 0.7527 | Regression_pt: 2.7185 | Regression_sin_phi: 1.6859 | LR: 3.78e-04 | DataLoad Time: 0.0028s | Model Forward Time: 3.2749s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.0598 | max_loss: 0.0036 | max_backward: 0.1881 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 2.1012 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5304.0 | uss_MB: 4326.3 | child_rss_MB: 10027.9 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 88.9 +rank6 [2026-10-27 17:07:14,836] INFO : Step 17000/100000 rank6 | Train Loss: 13.6189 | Train Loss Unweighted: 4.8180 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7348 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4756 | Regression_eta: 0.0731 | Regression_pt: 0.4225 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.8095 | physical_nopu: 4.7681 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2660 | Classification_binary: 3.4633 | Regression_cos_phi: 1.2107 | Regression_energy: 2.9089 | Regression_eta: 0.5358 | Regression_pt: 3.8984 | Regression_sin_phi: 1.2855 | LR: 3.75e-04 | DataLoad Time: 0.0081s | Model Forward Time: 4.0343s | Phase Times: max_pre_step_barrier: 0.6520 | max_batch_to_device: 0.0829 | max_forward: 0.1388 | max_loss: 0.0036 | max_backward: 0.0636 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 2.2240 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5273.0 | uss_MB: 4329.2 | child_rss_MB: 9776.7 | allocated_MB: 394.3 | reserved_MB: 12168.0 | inactive_split_MB: 88.4 +rank6 [2026-10-27 17:10:19,258] INFO : Step 18000/100000 rank6 | Train Loss: 12.6032 | Train Loss Unweighted: 4.6266 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7426 | Regression_cos_phi: 0.0507 | Regression_energy: 0.3730 | Regression_eta: 0.0736 | Regression_pt: 0.3195 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7954 | physical_nopu: 4.7518 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2088 | Classification_binary: 3.6601 | Regression_cos_phi: 1.6002 | Regression_energy: 1.7649 | Regression_eta: 0.7813 | Regression_pt: 2.1390 | Regression_sin_phi: 1.8578 | LR: 3.72e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.1882s | Phase Times: max_pre_step_barrier: 0.0217 | max_batch_to_device: 0.0009 | max_forward: 0.1212 | max_loss: 0.0040 | max_backward: 0.1670 | max_optimizer_step: 0.1610 | max_tensorboard_logging: 3.0399 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5245.9 | uss_MB: 4331.6 | child_rss_MB: 9623.0 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 87.1 +rank6 [2026-10-27 17:13:21,531] INFO : Step 19000/100000 rank6 | Train Loss: 14.6123 | Train Loss Unweighted: 5.0036 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.6735 | Regression_cos_phi: 0.0398 | Regression_energy: 0.6019 | Regression_eta: 0.0612 | Regression_pt: 0.5669 | Regression_sin_phi: 0.0441 | Group Train Loss Unweighted (window mean): physical_pu: 3.7887 | physical_nopu: 4.7426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3110 | Classification_binary: 3.7211 | Regression_cos_phi: 1.4230 | Regression_energy: 2.6413 | Regression_eta: 0.6536 | Regression_pt: 3.6440 | Regression_sin_phi: 1.6631 | LR: 3.69e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.4963s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1593 | max_loss: 0.0094 | max_backward: 0.0749 | max_optimizer_step: 0.1658 | max_tensorboard_logging: 2.3024 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5218.9 | uss_MB: 4333.7 | child_rss_MB: 9466.4 | allocated_MB: 394.5 | reserved_MB: 12168.0 | inactive_split_MB: 103.1 +rank6 [2026-10-27 17:16:34,024] INFO : Step 20000/100000 rank6 | Train Loss: 13.1046 | Train Loss Unweighted: 4.7571 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.7799 | Regression_cos_phi: 0.0517 | Regression_energy: 0.4251 | Regression_eta: 0.0654 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0511 | Group Train Loss Unweighted (window mean): physical_pu: 3.7835 | physical_nopu: 4.7206 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2854 | Classification_binary: 3.8070 | Regression_cos_phi: 2.0664 | Regression_energy: 2.4369 | Regression_eta: 0.8088 | Regression_pt: 3.1210 | Regression_sin_phi: 2.0141 | LR: 3.66e-04 | DataLoad Time: 0.0031s | Model Forward Time: 3.2223s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0008 | max_forward: 0.1344 | max_loss: 0.0037 | max_backward: 0.0763 | max_optimizer_step: 0.1649 | max_tensorboard_logging: 1.9693 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5216.3 | uss_MB: 4336.2 | child_rss_MB: 9462.6 | allocated_MB: 392.8 | reserved_MB: 12168.0 | inactive_split_MB: 89.4 +rank6 [2026-10-27 17:16:34,969] INFO : Running validation on rank6 +rank6 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 17:18:27,486] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 17:18:27,879] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 17:18:53,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 17:18:54,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 17:18:54,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 17:18:55,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 17:18:55,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 17:18:56,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 17:18:56,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 17:18:57,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 17:19:07,254] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 17:19:07,312] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 17:19:07,733] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 17:19:22,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 17:19:23,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 17:19:23,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 17:19:24,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 17:19:24,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 17:19:25,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 17:19:25,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 17:19:26,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 17:19:36,235] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 17:19:36,428] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 17:19:36,812] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 17:19:51,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 17:19:52,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 17:19:52,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 17:19:53,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 17:19:53,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 17:19:54,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 17:19:54,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 17:19:55,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 17:20:05,076] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-27 17:23:25,419] INFO : Step 21000/100000 rank6 | Train Loss: 12.4320 | Train Loss Unweighted: 4.5235 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.6041 | Regression_cos_phi: 0.0444 | Regression_energy: 0.3902 | Regression_eta: 0.0812 | Regression_pt: 0.3383 | Regression_sin_phi: 0.0478 | Group Train Loss Unweighted (window mean): physical_pu: 3.7732 | physical_nopu: 4.7052 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3417 | Classification_binary: 3.6530 | Regression_cos_phi: 1.4339 | Regression_energy: 2.2682 | Regression_eta: 0.8816 | Regression_pt: 3.0601 | Regression_sin_phi: 1.2810 | LR: 3.62e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.7291s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.2105 | max_loss: 0.0056 | max_backward: 0.1347 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 2.3606 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5326.2 | uss_MB: 4454.3 | child_rss_MB: 9450.4 | allocated_MB: 394.6 | reserved_MB: 11062.0 | inactive_split_MB: 89.8 +rank6 [2026-10-27 17:26:14,701] INFO : Step 22000/100000 rank6 | Train Loss: 14.5232 | Train Loss Unweighted: 4.8893 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.6034 | Regression_cos_phi: 0.0476 | Regression_energy: 0.5714 | Regression_eta: 0.0738 | Regression_pt: 0.5351 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.7709 | physical_nopu: 4.6938 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3635 | Classification_binary: 3.6478 | Regression_cos_phi: 1.6904 | Regression_energy: 3.0237 | Regression_eta: 0.6834 | Regression_pt: 4.1250 | Regression_sin_phi: 1.3543 | LR: 3.59e-04 | DataLoad Time: 0.0028s | Model Forward Time: 2.6768s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0010 | max_forward: 0.0774 | max_loss: 0.0142 | max_backward: 0.0921 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.4442 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1027 | Memory: rss_MB: 5327.2 | uss_MB: 4458.5 | child_rss_MB: 9447.9 | allocated_MB: 394.2 | reserved_MB: 11488.0 | inactive_split_MB: 86.8 +rank6 [2026-10-27 17:29:50,014] INFO : Step 23000/100000 rank6 | Train Loss: 12.2140 | Train Loss Unweighted: 4.5907 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.7888 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3388 | Regression_eta: 0.0762 | Regression_pt: 0.2687 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.7690 | physical_nopu: 4.6810 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2164 | Classification_binary: 3.8612 | Regression_cos_phi: 2.0770 | Regression_energy: 1.5153 | Regression_eta: 0.8017 | Regression_pt: 1.7098 | Regression_sin_phi: 2.0535 | LR: 3.55e-04 | DataLoad Time: 0.0108s | Model Forward Time: 4.1180s | Phase Times: max_pre_step_barrier: 0.7590 | max_batch_to_device: 0.0007 | max_forward: 0.0763 | max_loss: 0.0189 | max_backward: 0.1117 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 2.0913 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5325.2 | uss_MB: 4459.8 | child_rss_MB: 9460.6 | allocated_MB: 394.0 | reserved_MB: 12220.0 | inactive_split_MB: 178.1 +rank6 [2026-10-27 17:33:22,189] INFO : Step 24000/100000 rank6 | Train Loss: 15.4786 | Train Loss Unweighted: 4.9908 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.5301 | Regression_cos_phi: 0.0467 | Regression_energy: 0.6682 | Regression_eta: 0.0624 | Regression_pt: 0.6223 | Regression_sin_phi: 0.0450 | Group Train Loss Unweighted (window mean): physical_pu: 3.7564 | physical_nopu: 4.6761 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3145 | Classification_binary: 3.6372 | Regression_cos_phi: 1.8091 | Regression_energy: 2.5893 | Regression_eta: 0.7557 | Regression_pt: 3.3828 | Regression_sin_phi: 1.7075 | LR: 3.51e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.9315s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0719 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1672 | max_tensorboard_logging: 1.8279 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5318.8 | uss_MB: 4462.5 | child_rss_MB: 9451.5 | allocated_MB: 393.8 | reserved_MB: 12220.0 | inactive_split_MB: 105.1 +rank6 [2026-10-27 17:36:26,064] INFO : Step 25000/100000 rank6 | Train Loss: 12.1246 | Train Loss Unweighted: 4.4783 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.7007 | Regression_cos_phi: 0.0551 | Regression_energy: 0.3196 | Regression_eta: 0.0694 | Regression_pt: 0.2642 | Regression_sin_phi: 0.0578 | Group Train Loss Unweighted (window mean): physical_pu: 3.7531 | physical_nopu: 4.6582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1775 | Classification_binary: 3.7673 | Regression_cos_phi: 1.7191 | Regression_energy: 1.2912 | Regression_eta: 0.7909 | Regression_pt: 1.5425 | Regression_sin_phi: 1.8392 | LR: 3.47e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.0302s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.1917 | max_loss: 0.0039 | max_backward: 0.1029 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 5304.8 | uss_MB: 4464.1 | child_rss_MB: 9445.8 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 86.4 +rank6 [2026-10-27 17:39:30,100] INFO : Step 26000/100000 rank6 | Train Loss: 12.6920 | Train Loss Unweighted: 4.5479 | Train Loss Components: Classification: 0.0188 | Classification_binary: 3.6003 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4034 | Regression_eta: 0.0734 | Regression_pt: 0.3564 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.7511 | physical_nopu: 4.6505 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3540 | Classification_binary: 3.4724 | Regression_cos_phi: 1.8822 | Regression_energy: 1.8555 | Regression_eta: 0.8249 | Regression_pt: 2.3157 | Regression_sin_phi: 1.7702 | LR: 3.43e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.0941s | Phase Times: max_pre_step_barrier: 0.0479 | max_batch_to_device: 0.0008 | max_forward: 0.1278 | max_loss: 0.0037 | max_backward: 0.0877 | max_optimizer_step: 0.1868 | max_tensorboard_logging: 1.8908 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5292.2 | uss_MB: 4467.2 | child_rss_MB: 9407.7 | allocated_MB: 395.0 | reserved_MB: 12220.0 | inactive_split_MB: 115.0 +rank6 [2026-10-27 17:42:32,922] INFO : Step 27000/100000 rank6 | Train Loss: 16.2758 | Train Loss Unweighted: 5.1421 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5608 | Regression_cos_phi: 0.0448 | Regression_energy: 0.7180 | Regression_eta: 0.0780 | Regression_pt: 0.6791 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.7443 | physical_nopu: 4.6426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3205 | Classification_binary: 3.5627 | Regression_cos_phi: 1.4466 | Regression_energy: 2.8408 | Regression_eta: 0.7480 | Regression_pt: 3.8466 | Regression_sin_phi: 1.4380 | LR: 3.39e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.0074s | Phase Times: max_pre_step_barrier: 0.0385 | max_batch_to_device: 0.0009 | max_forward: 0.0708 | max_loss: 0.0038 | max_backward: 0.0914 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.8878 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5283.2 | uss_MB: 4469.8 | child_rss_MB: 9400.6 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 87.9 +rank6 [2026-10-27 17:45:38,005] INFO : Step 28000/100000 rank6 | Train Loss: 11.9115 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6175 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3339 | Regression_eta: 0.0761 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.7393 | physical_nopu: 4.6288 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3026 | Classification_binary: 3.4998 | Regression_cos_phi: 1.4771 | Regression_energy: 1.3186 | Regression_eta: 0.8566 | Regression_pt: 1.5864 | Regression_sin_phi: 1.1862 | LR: 3.35e-04 | DataLoad Time: 0.0021s | Model Forward Time: 4.8021s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0008 | max_forward: 0.0886 | max_loss: 0.0038 | max_backward: 0.1193 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 3.6285 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 5274.2 | uss_MB: 4146.2 | child_rss_MB: 9355.1 | allocated_MB: 394.0 | reserved_MB: 12414.0 | inactive_split_MB: 106.5 +rank6 [2026-10-27 17:48:54,715] INFO : Step 29000/100000 rank6 | Train Loss: 13.9221 | Train Loss Unweighted: 4.6615 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.5075 | Regression_cos_phi: 0.0491 | Regression_energy: 0.5008 | Regression_eta: 0.0786 | Regression_pt: 0.4589 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7405 | physical_nopu: 4.6233 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3733 | Classification_binary: 3.6204 | Regression_cos_phi: 2.0066 | Regression_energy: 1.6647 | Regression_eta: 0.9238 | Regression_pt: 2.0043 | Regression_sin_phi: 1.6394 | LR: 3.30e-04 | DataLoad Time: 0.0050s | Model Forward Time: 3.3781s | Phase Times: max_pre_step_barrier: 0.1677 | max_batch_to_device: 0.0009 | max_forward: 0.0754 | max_loss: 0.0038 | max_backward: 0.0825 | max_optimizer_step: 0.1673 | max_tensorboard_logging: 2.1272 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5252.2 | uss_MB: 4453.2 | child_rss_MB: 9260.7 | allocated_MB: 394.9 | reserved_MB: 12414.0 | inactive_split_MB: 107.1 +rank6 [2026-10-27 17:51:57,834] INFO : Step 30000/100000 rank6 | Train Loss: 12.4988 | Train Loss Unweighted: 4.5639 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.6431 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3981 | Regression_eta: 0.0722 | Regression_pt: 0.3405 | Regression_sin_phi: 0.0498 | Group Train Loss Unweighted (window mean): physical_pu: 3.7342 | physical_nopu: 4.6125 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2394 | Classification_binary: 3.7165 | Regression_cos_phi: 1.7167 | Regression_energy: 1.4573 | Regression_eta: 0.7877 | Regression_pt: 1.7762 | Regression_sin_phi: 1.8091 | LR: 3.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.4592s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0762 | max_loss: 0.0136 | max_backward: 0.0854 | max_optimizer_step: 0.1722 | max_tensorboard_logging: 1.3349 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5242.2 | uss_MB: 4475.0 | child_rss_MB: 9193.8 | allocated_MB: 393.7 | reserved_MB: 12480.0 | inactive_split_MB: 89.7 +rank6 [2026-10-27 17:51:58,780] INFO : Running validation on rank6 +rank6 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 17:51:58,781] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 17:53:53,313] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 17:53:53,723] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 17:54:18,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 17:54:18,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 17:54:19,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 17:54:19,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 17:54:20,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 17:54:20,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 17:54:21,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 17:54:21,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 17:54:31,802] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 17:54:31,840] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 17:54:32,213] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 17:54:47,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 17:54:47,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 17:54:48,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 17:54:48,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 17:54:49,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 17:54:49,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 17:54:50,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 17:54:50,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 17:55:00,544] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 17:55:00,637] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 17:55:01,040] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 17:55:16,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 17:55:17,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 17:55:17,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 17:55:18,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 17:55:18,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 17:55:19,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 17:55:19,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 17:55:20,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 17:55:30,078] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-27 17:58:47,285] INFO : Step 31000/100000 rank6 | Train Loss: 12.0027 | Train Loss Unweighted: 4.4503 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.5779 | Regression_cos_phi: 0.0460 | Regression_energy: 0.3731 | Regression_eta: 0.0704 | Regression_pt: 0.3230 | Regression_sin_phi: 0.0451 | Group Train Loss Unweighted (window mean): physical_pu: 3.7262 | physical_nopu: 4.6083 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2669 | Classification_binary: 3.5095 | Regression_cos_phi: 1.4537 | Regression_energy: 1.5940 | Regression_eta: 0.7116 | Regression_pt: 1.8890 | Regression_sin_phi: 1.2988 | LR: 3.21e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.5859s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.1458 | max_loss: 0.0037 | max_backward: 0.1256 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 2.3030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7018.1 | uss_MB: 6249.8 | child_rss_MB: 9197.9 | allocated_MB: 394.8 | reserved_MB: 11168.0 | inactive_split_MB: 96.2 +rank6 [2026-10-27 18:01:54,932] INFO : Step 32000/100000 rank6 | Train Loss: 13.5217 | Train Loss Unweighted: 4.6918 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.5357 | Regression_cos_phi: 0.0443 | Regression_energy: 0.5169 | Regression_eta: 0.0728 | Regression_pt: 0.4649 | Regression_sin_phi: 0.0409 | Group Train Loss Unweighted (window mean): physical_pu: 3.7264 | physical_nopu: 4.6006 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3634 | Classification_binary: 3.5154 | Regression_cos_phi: 1.7500 | Regression_energy: 2.7379 | Regression_eta: 1.2581 | Regression_pt: 3.3247 | Regression_sin_phi: 1.8755 | LR: 3.16e-04 | DataLoad Time: 0.0080s | Model Forward Time: 2.3903s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.0795 | max_loss: 0.0917 | max_backward: 0.1851 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.1890 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5310.0 | uss_MB: 4562.1 | child_rss_MB: 9157.4 | allocated_MB: 394.9 | reserved_MB: 11240.0 | inactive_split_MB: 92.3 +rank6 [2026-10-27 18:04:57,678] INFO : Step 33000/100000 rank6 | Train Loss: 11.0126 | Train Loss Unweighted: 3.7465 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9928 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2905 | Regression_eta: 0.1058 | Regression_pt: 0.2472 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.7276 | physical_nopu: 4.5909 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3406 | Classification_binary: 3.0570 | Regression_cos_phi: 1.7883 | Regression_energy: 1.5080 | Regression_eta: 1.1690 | Regression_pt: 1.9732 | Regression_sin_phi: 1.6231 | LR: 3.12e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.8904s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0027 | max_forward: 0.1127 | max_loss: 0.0150 | max_backward: 0.3396 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.4660 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5295.8 | uss_MB: 4291.2 | child_rss_MB: 9057.9 | allocated_MB: 404.8 | reserved_MB: 11922.0 | inactive_split_MB: 135.4 +rank6 [2026-10-27 18:07:50,121] INFO : Step 34000/100000 rank6 | Train Loss: 14.2400 | Train Loss Unweighted: 4.7634 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5051 | Regression_cos_phi: 0.0450 | Regression_energy: 0.5627 | Regression_eta: 0.0631 | Regression_pt: 0.5251 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7180 | physical_nopu: 4.5858 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3898 | Classification_binary: 3.4032 | Regression_cos_phi: 1.3666 | Regression_energy: 3.3708 | Regression_eta: 0.6936 | Regression_pt: 4.9375 | Regression_sin_phi: 1.3440 | LR: 3.07e-04 | DataLoad Time: 0.0010s | Model Forward Time: 2.7417s | Phase Times: max_pre_step_barrier: 0.1802 | max_batch_to_device: 0.0765 | max_forward: 0.1650 | max_loss: 0.0036 | max_backward: 0.0935 | max_optimizer_step: 0.2788 | max_tensorboard_logging: 1.2621 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5285.5 | uss_MB: 4563.5 | child_rss_MB: 9059.3 | allocated_MB: 395.5 | reserved_MB: 11922.0 | inactive_split_MB: 99.3 +rank6 [2026-10-27 18:10:53,637] INFO : Step 35000/100000 rank6 | Train Loss: 11.4005 | Train Loss Unweighted: 4.4142 | Train Loss Components: Classification: 0.0091 | Classification_binary: 3.6972 | Regression_cos_phi: 0.0530 | Regression_energy: 0.2984 | Regression_eta: 0.0671 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.7147 | physical_nopu: 4.5802 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2267 | Classification_binary: 3.7012 | Regression_cos_phi: 1.9323 | Regression_energy: 1.4226 | Regression_eta: 0.7361 | Regression_pt: 1.7467 | Regression_sin_phi: 1.7572 | LR: 3.02e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.4162s | Phase Times: max_pre_step_barrier: 0.0682 | max_batch_to_device: 0.0008 | max_forward: 0.1243 | max_loss: 0.0038 | max_backward: 0.0883 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.1949 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5274.5 | uss_MB: 4566.4 | child_rss_MB: 9063.3 | allocated_MB: 393.8 | reserved_MB: 11922.0 | inactive_split_MB: 85.2 +rank6 [2026-10-27 18:13:50,578] INFO : Step 36000/100000 rank6 | Train Loss: 11.8438 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.5751 | Regression_cos_phi: 0.0431 | Regression_energy: 0.3555 | Regression_eta: 0.0641 | Regression_pt: 0.3210 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.7110 | physical_nopu: 4.5729 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2963 | Classification_binary: 3.6987 | Regression_cos_phi: 1.4503 | Regression_energy: 1.5606 | Regression_eta: 0.6618 | Regression_pt: 2.1552 | Regression_sin_phi: 1.4627 | LR: 2.97e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4513s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0993 | max_loss: 0.0066 | max_backward: 0.1760 | max_optimizer_step: 0.1735 | max_tensorboard_logging: 1.2166 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 5259.5 | uss_MB: 4568.3 | child_rss_MB: 9032.9 | allocated_MB: 395.0 | reserved_MB: 11922.0 | inactive_split_MB: 95.2 +rank6 [2026-10-27 18:16:45,596] INFO : Step 37000/100000 rank6 | Train Loss: 15.1390 | Train Loss Unweighted: 4.8166 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4396 | Regression_cos_phi: 0.0489 | Regression_energy: 0.6142 | Regression_eta: 0.0886 | Regression_pt: 0.5621 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7102 | physical_nopu: 4.5673 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4065 | Classification_binary: 3.4287 | Regression_cos_phi: 1.9884 | Regression_energy: 2.1037 | Regression_eta: 1.0005 | Regression_pt: 2.9172 | Regression_sin_phi: 1.8657 | LR: 2.91e-04 | DataLoad Time: 0.0013s | Model Forward Time: 2.2791s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1889 | max_loss: 0.0036 | max_backward: 0.0899 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 1.0901 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 5243.2 | uss_MB: 4569.9 | child_rss_MB: 9022.4 | allocated_MB: 394.3 | reserved_MB: 11922.0 | inactive_split_MB: 107.1 +rank6 [2026-10-27 18:19:42,075] INFO : Step 38000/100000 rank6 | Train Loss: 11.6256 | Train Loss Unweighted: 4.4424 | Train Loss Components: Classification: 0.0117 | Classification_binary: 3.6501 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3369 | Regression_eta: 0.0664 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7131 | physical_nopu: 4.5682 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2961 | Classification_binary: 3.7874 | Regression_cos_phi: 1.6384 | Regression_energy: 1.4334 | Regression_eta: 0.7269 | Regression_pt: 1.6656 | Regression_sin_phi: 1.6171 | LR: 2.86e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.2500s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.0965 | max_loss: 0.0038 | max_backward: 0.1051 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5237.2 | uss_MB: 4571.0 | child_rss_MB: 9030.3 | allocated_MB: 393.3 | reserved_MB: 12082.0 | inactive_split_MB: 110.5 +rank6 [2026-10-27 18:22:37,812] INFO : Step 39000/100000 rank6 | Train Loss: 13.0208 | Train Loss Unweighted: 4.4928 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4758 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4360 | Regression_eta: 0.0793 | Regression_pt: 0.3879 | Regression_sin_phi: 0.0500 | Group Train Loss Unweighted (window mean): physical_pu: 3.7060 | physical_nopu: 4.5607 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3166 | Classification_binary: 3.4054 | Regression_cos_phi: 1.5792 | Regression_energy: 2.6871 | Regression_eta: 0.9314 | Regression_pt: 3.4610 | Regression_sin_phi: 1.8556 | LR: 2.81e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.2316s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.0864 | max_loss: 0.0038 | max_backward: 0.0877 | max_optimizer_step: 0.1700 | max_tensorboard_logging: 1.1418 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5224.7 | uss_MB: 4572.8 | child_rss_MB: 9008.2 | allocated_MB: 394.4 | reserved_MB: 12706.0 | inactive_split_MB: 84.9 +rank6 [2026-10-27 18:25:31,277] INFO : Step 40000/100000 rank6 | Train Loss: 14.7562 | Train Loss Unweighted: 4.9007 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5708 | Regression_cos_phi: 0.0459 | Regression_energy: 0.5954 | Regression_eta: 0.0747 | Regression_pt: 0.5553 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7006 | physical_nopu: 4.5579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2964 | Classification_binary: 3.4246 | Regression_cos_phi: 1.9642 | Regression_energy: 2.5235 | Regression_eta: 1.0777 | Regression_pt: 3.9565 | Regression_sin_phi: 1.3701 | LR: 2.76e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.3072s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.0814 | max_loss: 0.0614 | max_backward: 0.1417 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 1.1378 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5222.7 | uss_MB: 4573.6 | child_rss_MB: 8992.8 | allocated_MB: 393.7 | reserved_MB: 12706.0 | inactive_split_MB: 107.7 +rank6 [2026-10-27 18:25:32,321] INFO : Running validation on rank6 +rank6 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 18:27:23,860] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 18:27:24,220] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 18:27:48,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 18:27:48,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 18:27:49,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 18:27:49,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 18:27:50,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 18:27:50,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 18:27:51,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 18:27:51,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 18:28:01,690] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 18:28:01,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 18:28:02,235] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 18:28:17,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 18:28:18,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 18:28:18,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 18:28:19,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 18:28:19,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 18:28:20,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 18:28:20,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 18:28:20,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 18:28:30,982] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 18:28:30,989] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 18:28:31,366] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 18:28:46,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 18:28:47,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 18:28:47,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 18:28:47,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 18:28:48,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 18:28:48,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 18:28:49,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 18:28:49,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 18:28:59,726] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 18:32:27,442] INFO : Step 41000/100000 rank6 | Train Loss: 12.3609 | Train Loss Unweighted: 4.4817 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5838 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3837 | Regression_eta: 0.0731 | Regression_pt: 0.3287 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.7016 | physical_nopu: 4.5433 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3627 | Classification_binary: 3.4264 | Regression_cos_phi: 1.5523 | Regression_energy: 1.6474 | Regression_eta: 0.5220 | Regression_pt: 2.1960 | Regression_sin_phi: 1.7822 | LR: 2.70e-04 | DataLoad Time: 0.0025s | Model Forward Time: 4.5166s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0009 | max_forward: 0.1338 | max_loss: 0.0038 | max_backward: 0.1857 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 3.1120 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 7028.8 | uss_MB: 6392.1 | child_rss_MB: 8998.3 | allocated_MB: 394.0 | reserved_MB: 11364.0 | inactive_split_MB: 106.1 +rank6 [2026-10-27 18:35:19,857] INFO : Step 42000/100000 rank6 | Train Loss: 12.9790 | Train Loss Unweighted: 4.5354 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4379 | Regression_cos_phi: 0.0458 | Regression_energy: 0.4856 | Regression_eta: 0.0625 | Regression_pt: 0.4500 | Regression_sin_phi: 0.0370 | Group Train Loss Unweighted (window mean): physical_pu: 3.6959 | physical_nopu: 4.5474 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3709 | Classification_binary: 3.5592 | Regression_cos_phi: 1.1845 | Regression_energy: 2.1331 | Regression_eta: 0.7310 | Regression_pt: 2.8546 | Regression_sin_phi: 1.7297 | LR: 2.65e-04 | DataLoad Time: 0.0115s | Model Forward Time: 2.6324s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0009 | max_forward: 0.1375 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 1.3492 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 7016.8 | uss_MB: 5332.4 | child_rss_MB: 8994.4 | allocated_MB: 394.2 | reserved_MB: 12150.0 | inactive_split_MB: 99.3 +rank6 [2026-10-27 18:38:35,926] INFO : Step 43000/100000 rank6 | Train Loss: 10.9006 | Train Loss Unweighted: 3.7057 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.9740 | Regression_cos_phi: 0.0473 | Regression_energy: 0.2786 | Regression_eta: 0.1094 | Regression_pt: 0.2322 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6942 | physical_nopu: 4.5421 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3456 | Classification_binary: 2.9440 | Regression_cos_phi: 1.6963 | Regression_energy: 1.3444 | Regression_eta: 1.1561 | Regression_pt: 1.6609 | Regression_sin_phi: 1.6035 | LR: 2.59e-04 | DataLoad Time: 0.0018s | Model Forward Time: 5.2294s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0039 | max_forward: 0.1100 | max_loss: 0.0637 | max_backward: 0.2303 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 3.9250 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5285.0 | uss_MB: 4686.1 | child_rss_MB: 8875.5 | allocated_MB: 406.8 | reserved_MB: 12150.0 | inactive_split_MB: 152.7 +rank6 [2026-10-27 18:41:39,775] INFO : Step 44000/100000 rank6 | Train Loss: 14.8629 | Train Loss Unweighted: 4.8399 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4856 | Regression_cos_phi: 0.0438 | Regression_energy: 0.6070 | Regression_eta: 0.0739 | Regression_pt: 0.5644 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6933 | physical_nopu: 4.5331 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3930 | Classification_binary: 3.2459 | Regression_cos_phi: 1.2780 | Regression_energy: 2.3862 | Regression_eta: 0.6081 | Regression_pt: 3.3537 | Regression_sin_phi: 1.3347 | LR: 2.54e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.6836s | Phase Times: max_pre_step_barrier: 0.0851 | max_batch_to_device: 0.0008 | max_forward: 0.0946 | max_loss: 0.0041 | max_backward: 0.0964 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.3558 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5284.0 | uss_MB: 4679.1 | child_rss_MB: 8885.5 | allocated_MB: 394.8 | reserved_MB: 12150.0 | inactive_split_MB: 94.9 +rank6 [2026-10-27 18:44:46,984] INFO : Step 45000/100000 rank6 | Train Loss: 12.9535 | Train Loss Unweighted: 4.5658 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.6115 | Regression_cos_phi: 0.0537 | Regression_energy: 0.4117 | Regression_eta: 0.0846 | Regression_pt: 0.3413 | Regression_sin_phi: 0.0531 | Group Train Loss Unweighted (window mean): physical_pu: 3.6892 | physical_nopu: 4.5283 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2491 | Classification_binary: 3.6222 | Regression_cos_phi: 2.0506 | Regression_energy: 1.5022 | Regression_eta: 0.7786 | Regression_pt: 1.7488 | Regression_sin_phi: 1.8551 | LR: 2.48e-04 | DataLoad Time: 0.0013s | Model Forward Time: 2.4276s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1077 | max_loss: 0.0040 | max_backward: 0.1457 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 1.2404 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 5275.4 | uss_MB: 4680.1 | child_rss_MB: 8893.2 | allocated_MB: 394.0 | reserved_MB: 13366.0 | inactive_split_MB: 74.2 +rank6 [2026-10-27 18:48:06,241] INFO : Step 46000/100000 rank6 | Train Loss: 12.3783 | Train Loss Unweighted: 4.4476 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.5252 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3923 | Regression_eta: 0.0679 | Regression_pt: 0.3506 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6869 | physical_nopu: 4.5281 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3314 | Classification_binary: 3.6270 | Regression_cos_phi: 1.5365 | Regression_energy: 1.8108 | Regression_eta: 0.7225 | Regression_pt: 2.3552 | Regression_sin_phi: 2.0817 | LR: 2.43e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4374s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1021 | max_loss: 0.0036 | max_backward: 0.1895 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 1.1894 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 5269.4 | uss_MB: 4681.7 | child_rss_MB: 8897.5 | allocated_MB: 395.1 | reserved_MB: 13366.0 | inactive_split_MB: 103.1 +rank6 [2026-10-27 18:51:16,142] INFO : Step 47000/100000 rank6 | Train Loss: 14.5142 | Train Loss Unweighted: 4.7631 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.3922 | Regression_cos_phi: 0.0407 | Regression_energy: 0.6220 | Regression_eta: 0.0662 | Regression_pt: 0.5862 | Regression_sin_phi: 0.0406 | Group Train Loss Unweighted (window mean): physical_pu: 3.6853 | physical_nopu: 4.5234 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2614 | Classification_binary: 3.4429 | Regression_cos_phi: 1.8509 | Regression_energy: 1.8397 | Regression_eta: 0.7095 | Regression_pt: 2.4913 | Regression_sin_phi: 1.5885 | LR: 2.37e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.5379s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0667 | max_forward: 0.1426 | max_loss: 0.0037 | max_backward: 0.0873 | max_optimizer_step: 0.2591 | max_tensorboard_logging: 1.1650 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5267.4 | uss_MB: 4682.3 | child_rss_MB: 8896.3 | allocated_MB: 394.9 | reserved_MB: 14166.0 | inactive_split_MB: 95.2 +rank6 [2026-10-27 18:54:12,871] INFO : Step 48000/100000 rank6 | Train Loss: 11.2030 | Train Loss Unweighted: 4.3343 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.6447 | Regression_cos_phi: 0.0521 | Regression_energy: 0.2822 | Regression_eta: 0.0657 | Regression_pt: 0.2293 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6844 | physical_nopu: 4.5168 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1749 | Classification_binary: 3.8220 | Regression_cos_phi: 1.8076 | Regression_energy: 1.5609 | Regression_eta: 0.9638 | Regression_pt: 1.7518 | Regression_sin_phi: 1.8563 | LR: 2.31e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.2609s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.0058 | max_backward: 0.0852 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1613 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5253.4 | uss_MB: 4683.3 | child_rss_MB: 8890.9 | allocated_MB: 393.9 | reserved_MB: 14166.0 | inactive_split_MB: 101.2 +rank6 [2026-10-27 18:57:19,782] INFO : Step 49000/100000 rank6 | Train Loss: 12.4372 | Train Loss Unweighted: 4.4483 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.5320 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3881 | Regression_eta: 0.0742 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6801 | physical_nopu: 4.5174 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3916 | Classification_binary: 3.6067 | Regression_cos_phi: 1.8554 | Regression_energy: 1.7876 | Regression_eta: 0.8466 | Regression_pt: 2.3165 | Regression_sin_phi: 1.5414 | LR: 2.26e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.4738s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0009 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1014 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 1.3340 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5240.2 | uss_MB: 4683.4 | child_rss_MB: 8815.5 | allocated_MB: 393.7 | reserved_MB: 14166.0 | inactive_split_MB: 100.8 +rank6 [2026-10-27 19:00:32,233] INFO : Step 50000/100000 rank6 | Train Loss: 13.3654 | Train Loss Unweighted: 4.5742 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4424 | Regression_cos_phi: 0.0426 | Regression_energy: 0.5013 | Regression_eta: 0.0683 | Regression_pt: 0.4573 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6814 | physical_nopu: 4.5132 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3992 | Classification_binary: 3.3624 | Regression_cos_phi: 1.6625 | Regression_energy: 2.1537 | Regression_eta: 0.8432 | Regression_pt: 3.0064 | Regression_sin_phi: 1.8326 | LR: 2.20e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.5859s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0010 | max_forward: 0.1892 | max_loss: 0.0115 | max_backward: 0.1333 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.2641 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0862 | Memory: rss_MB: 5234.2 | uss_MB: 4685.5 | child_rss_MB: 8779.3 | allocated_MB: 394.6 | reserved_MB: 14166.0 | inactive_split_MB: 91.2 +rank6 [2026-10-27 19:00:33,174] INFO : Running validation on rank6 +rank6 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 19:02:37,502] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 19:02:37,941] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 19:03:02,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 19:03:03,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 19:03:03,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 19:03:04,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 19:03:04,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 19:03:05,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 19:03:05,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 19:03:06,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 19:03:16,065] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 19:03:16,262] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 19:03:16,635] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 19:03:31,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 19:03:32,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 19:03:32,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 19:03:33,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 19:03:33,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 19:03:33,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 19:03:34,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 19:03:34,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 19:03:44,862] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 19:03:45,438] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 19:03:45,816] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 19:04:01,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 19:04:01,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 19:04:02,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 19:04:02,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 19:04:03,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 19:04:03,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 19:04:03,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 19:04:04,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 19:04:14,385] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 19:07:44,424] INFO : Step 51000/100000 rank6 | Train Loss: 12.0993 | Train Loss Unweighted: 4.3946 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5485 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3591 | Regression_eta: 0.0789 | Regression_pt: 0.2941 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6745 | physical_nopu: 4.5037 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3734 | Classification_binary: 3.3602 | Regression_cos_phi: 1.6942 | Regression_energy: 1.5367 | Regression_eta: 0.5632 | Regression_pt: 1.8673 | Regression_sin_phi: 1.9113 | LR: 2.14e-04 | DataLoad Time: 0.0083s | Model Forward Time: 3.3992s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0008 | max_forward: 0.1391 | max_loss: 0.0036 | max_backward: 0.1509 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 2.0683 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 7102.6 | uss_MB: 6555.6 | child_rss_MB: 8775.1 | allocated_MB: 393.6 | reserved_MB: 10952.0 | inactive_split_MB: 101.2 +rank6 [2026-10-27 19:10:55,345] INFO : Step 52000/100000 rank6 | Train Loss: 10.7195 | Train Loss Unweighted: 3.6688 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9553 | Regression_cos_phi: 0.0470 | Regression_energy: 0.2719 | Regression_eta: 0.1059 | Regression_pt: 0.2258 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6784 | physical_nopu: 4.5078 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3421 | Classification_binary: 3.0511 | Regression_cos_phi: 1.7304 | Regression_energy: 1.3277 | Regression_eta: 1.1965 | Regression_pt: 1.5976 | Regression_sin_phi: 1.6812 | LR: 2.09e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.5040s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0031 | max_forward: 0.1853 | max_loss: 0.0256 | max_backward: 0.2604 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 2.0239 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7092.6 | uss_MB: 6564.5 | child_rss_MB: 8773.5 | allocated_MB: 404.8 | reserved_MB: 11626.0 | inactive_split_MB: 256.4 +rank6 [2026-10-27 19:13:48,854] INFO : Step 53000/100000 rank6 | Train Loss: 11.7022 | Train Loss Unweighted: 4.3546 | Train Loss Components: Classification: 0.0123 | Classification_binary: 3.5594 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3321 | Regression_eta: 0.0730 | Regression_pt: 0.2795 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6748 | physical_nopu: 4.5005 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2836 | Classification_binary: 3.4655 | Regression_cos_phi: 1.4064 | Regression_energy: 1.3964 | Regression_eta: 0.9333 | Regression_pt: 1.7389 | Regression_sin_phi: 1.4447 | LR: 2.03e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.1505s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0954 | max_loss: 0.0037 | max_backward: 0.1065 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.9892 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 5255.7 | uss_MB: 4729.4 | child_rss_MB: 8773.6 | allocated_MB: 393.6 | reserved_MB: 11910.0 | inactive_split_MB: 100.6 +rank6 [2026-10-27 19:17:18,773] INFO : Step 54000/100000 rank6 | Train Loss: 12.5762 | Train Loss Unweighted: 4.3665 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0487 | Regression_energy: 0.4104 | Regression_eta: 0.0739 | Regression_pt: 0.3685 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6696 | physical_nopu: 4.5036 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3757 | Classification_binary: 3.2351 | Regression_cos_phi: 1.9750 | Regression_energy: 2.1352 | Regression_eta: 0.6651 | Regression_pt: 2.7741 | Regression_sin_phi: 1.6103 | LR: 1.97e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.3959s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.1022 | max_forward: 0.1953 | max_loss: 0.0037 | max_backward: 0.0975 | max_optimizer_step: 0.2772 | max_tensorboard_logging: 2.0527 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5248.7 | uss_MB: 4730.3 | child_rss_MB: 8744.8 | allocated_MB: 394.9 | reserved_MB: 11910.0 | inactive_split_MB: 118.2 +rank6 [2026-10-27 19:20:28,222] INFO : Step 55000/100000 rank6 | Train Loss: 13.4632 | Train Loss Unweighted: 4.6500 | Train Loss Components: Classification: 0.0134 | Classification_binary: 3.5428 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4921 | Regression_eta: 0.0679 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6720 | physical_nopu: 4.4919 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2338 | Classification_binary: 3.6719 | Regression_cos_phi: 1.9385 | Regression_energy: 1.6869 | Regression_eta: 1.0534 | Regression_pt: 2.1312 | Regression_sin_phi: 1.8968 | LR: 1.92e-04 | DataLoad Time: 0.1556s | Model Forward Time: 3.6485s | Phase Times: max_pre_step_barrier: 0.1980 | max_batch_to_device: 0.0007 | max_forward: 0.0889 | max_loss: 0.0037 | max_backward: 0.2091 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 2.3779 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5235.8 | uss_MB: 4729.6 | child_rss_MB: 8640.7 | allocated_MB: 393.8 | reserved_MB: 11910.0 | inactive_split_MB: 89.1 +rank6 [2026-10-27 19:23:30,605] INFO : Step 56000/100000 rank6 | Train Loss: 11.6078 | Train Loss Unweighted: 4.2962 | Train Loss Components: Classification: 0.0140 | Classification_binary: 3.4759 | Regression_cos_phi: 0.0438 | Regression_energy: 0.3484 | Regression_eta: 0.0671 | Regression_pt: 0.2977 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6692 | physical_nopu: 4.4908 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3005 | Classification_binary: 3.4508 | Regression_cos_phi: 1.6084 | Regression_energy: 1.8721 | Regression_eta: 0.7909 | Regression_pt: 2.3754 | Regression_sin_phi: 1.4714 | LR: 1.86e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.2298s | Phase Times: max_pre_step_barrier: 0.0941 | max_batch_to_device: 0.0008 | max_forward: 0.0930 | max_loss: 0.0038 | max_backward: 0.0988 | max_optimizer_step: 0.1756 | max_tensorboard_logging: 1.8920 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 5221.8 | uss_MB: 4732.9 | child_rss_MB: 8582.9 | allocated_MB: 393.5 | reserved_MB: 11910.0 | inactive_split_MB: 115.7 +rank6 [2026-10-27 19:26:52,902] INFO : Step 57000/100000 rank6 | Train Loss: 13.9252 | Train Loss Unweighted: 4.6468 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4640 | Regression_cos_phi: 0.0464 | Regression_energy: 0.5144 | Regression_eta: 0.0781 | Regression_pt: 0.4791 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6687 | physical_nopu: 4.4875 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2689 | Classification_binary: 3.6401 | Regression_cos_phi: 1.3852 | Regression_energy: 2.0706 | Regression_eta: 0.5727 | Regression_pt: 2.7855 | Regression_sin_phi: 1.3776 | LR: 1.81e-04 | DataLoad Time: 0.0022s | Model Forward Time: 4.6261s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.1223 | max_forward: 0.2094 | max_loss: 0.0420 | max_backward: 0.1628 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 2.9072 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 5216.8 | uss_MB: 4734.2 | child_rss_MB: 8511.2 | allocated_MB: 394.3 | reserved_MB: 12240.0 | inactive_split_MB: 101.4 +rank6 [2026-10-27 19:30:07,652] INFO : Step 58000/100000 rank6 | Train Loss: 11.5910 | Train Loss Unweighted: 4.3047 | Train Loss Components: Classification: 0.0106 | Classification_binary: 3.5490 | Regression_cos_phi: 0.0544 | Regression_energy: 0.3108 | Regression_eta: 0.0631 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0510 | Group Train Loss Unweighted (window mean): physical_pu: 3.6630 | physical_nopu: 4.4842 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2617 | Classification_binary: 3.6197 | Regression_cos_phi: 1.8612 | Regression_energy: 1.5658 | Regression_eta: 0.6668 | Regression_pt: 2.0126 | Regression_sin_phi: 1.7231 | LR: 1.75e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.5121s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.1080 | max_backward: 0.1964 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.2749 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 5209.3 | uss_MB: 4734.4 | child_rss_MB: 8432.6 | allocated_MB: 393.9 | reserved_MB: 12240.0 | inactive_split_MB: 121.9 +rank6 [2026-10-27 19:33:12,149] INFO : Step 59000/100000 rank6 | Train Loss: 12.2218 | Train Loss Unweighted: 4.4115 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5166 | Regression_cos_phi: 0.0458 | Regression_energy: 0.3794 | Regression_eta: 0.0771 | Regression_pt: 0.3280 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6650 | physical_nopu: 4.4788 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3252 | Classification_binary: 3.4273 | Regression_cos_phi: 1.5325 | Regression_energy: 1.5282 | Regression_eta: 0.6954 | Regression_pt: 2.0458 | Regression_sin_phi: 1.4111 | LR: 1.70e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.2208s | Phase Times: max_pre_step_barrier: 0.6889 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0499 | max_backward: 0.1549 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5201.3 | uss_MB: 4735.4 | child_rss_MB: 8387.3 | allocated_MB: 394.0 | reserved_MB: 12240.0 | inactive_split_MB: 135.3 +rank6 [2026-10-27 19:36:26,517] INFO : Step 60000/100000 rank6 | Train Loss: 14.3810 | Train Loss Unweighted: 4.7229 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.4146 | Regression_cos_phi: 0.0449 | Regression_energy: 0.5841 | Regression_eta: 0.0716 | Regression_pt: 0.5495 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.6604 | physical_nopu: 4.4865 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3013 | Classification_binary: 3.3537 | Regression_cos_phi: 1.7491 | Regression_energy: 3.3416 | Regression_eta: 0.9773 | Regression_pt: 4.9413 | Regression_sin_phi: 1.9347 | LR: 1.64e-04 | DataLoad Time: 0.0029s | Model Forward Time: 2.4562s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0849 | max_loss: 0.0036 | max_backward: 0.1478 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 1.2880 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5191.9 | uss_MB: 4495.0 | child_rss_MB: 8328.9 | allocated_MB: 394.8 | reserved_MB: 12240.0 | inactive_split_MB: 87.0 +rank6 [2026-10-27 19:36:27,558] INFO : Running validation on rank6 +rank6 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 19:38:18,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 19:38:19,028] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 19:38:43,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 19:38:44,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 19:38:44,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 19:38:45,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 19:38:45,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 19:38:46,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 19:38:46,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 19:38:46,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 19:38:57,009] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 19:38:57,241] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 19:38:57,610] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 19:39:12,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 19:39:12,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 19:39:13,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 19:39:13,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 19:39:14,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 19:39:14,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 19:39:15,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 19:39:15,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 19:39:25,737] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 19:39:26,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 19:39:26,533] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 19:39:41,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 19:39:42,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 19:39:42,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 19:39:42,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 19:39:43,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 19:39:43,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 19:39:44,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 19:39:44,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 19:39:54,746] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-27 19:43:33,544] INFO : Step 61000/100000 rank6 | Train Loss: 11.4262 | Train Loss Unweighted: 4.3706 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.6500 | Regression_cos_phi: 0.0531 | Regression_energy: 0.2956 | Regression_eta: 0.0675 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6553 | physical_nopu: 4.4780 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2387 | Classification_binary: 3.5612 | Regression_cos_phi: 2.3637 | Regression_energy: 1.4062 | Regression_eta: 0.7962 | Regression_pt: 1.6028 | Regression_sin_phi: 1.7842 | LR: 1.59e-04 | DataLoad Time: 0.0065s | Model Forward Time: 3.1999s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1116 | max_loss: 0.0036 | max_backward: 0.1506 | max_optimizer_step: 0.1828 | max_tensorboard_logging: 1.9738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 5784.0 | uss_MB: 5329.2 | child_rss_MB: 8302.4 | allocated_MB: 393.5 | reserved_MB: 11190.0 | inactive_split_MB: 103.1 +rank6 [2026-10-27 19:46:49,589] INFO : Step 62000/100000 rank6 | Train Loss: 10.7781 | Train Loss Unweighted: 3.6510 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9253 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2791 | Regression_eta: 0.1043 | Regression_pt: 0.2316 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3292 | Classification_binary: 2.9794 | Regression_cos_phi: 1.7822 | Regression_energy: 1.4636 | Regression_eta: 1.1376 | Regression_pt: 1.8363 | Regression_sin_phi: 1.6039 | LR: 1.54e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.6556s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0030 | max_forward: 0.1294 | max_loss: 0.0038 | max_backward: 0.2007 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.2115 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0496 | Memory: rss_MB: 5775.0 | uss_MB: 5052.2 | child_rss_MB: 8186.6 | allocated_MB: 404.4 | reserved_MB: 11190.0 | inactive_split_MB: 148.6 +rank6 [2026-10-27 19:50:02,443] INFO : Step 63000/100000 rank6 | Train Loss: 11.5766 | Train Loss Unweighted: 4.3655 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6000 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3204 | Regression_eta: 0.0727 | Regression_pt: 0.2610 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6589 | physical_nopu: 4.4771 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3247 | Classification_binary: 3.7062 | Regression_cos_phi: 1.7645 | Regression_energy: 1.5365 | Regression_eta: 0.8675 | Regression_pt: 1.7089 | Regression_sin_phi: 1.6575 | LR: 1.49e-04 | DataLoad Time: 0.0013s | Model Forward Time: 2.3900s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.1036 | max_loss: 0.0037 | max_backward: 0.1140 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2208 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 5773.0 | uss_MB: 5331.6 | child_rss_MB: 8138.8 | allocated_MB: 393.8 | reserved_MB: 11190.0 | inactive_split_MB: 93.0 +rank6 [2026-10-27 19:53:17,771] INFO : Step 64000/100000 rank6 | Train Loss: 13.2709 | Train Loss Unweighted: 4.5542 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.4295 | Regression_cos_phi: 0.0429 | Regression_energy: 0.4948 | Regression_eta: 0.0672 | Regression_pt: 0.4587 | Regression_sin_phi: 0.0436 | Group Train Loss Unweighted (window mean): physical_pu: 3.6551 | physical_nopu: 4.4714 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3732 | Classification_binary: 3.3188 | Regression_cos_phi: 1.2802 | Regression_energy: 2.3775 | Regression_eta: 0.4311 | Regression_pt: 3.3256 | Regression_sin_phi: 1.4285 | LR: 1.43e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.3961s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0928 | max_loss: 0.0570 | max_backward: 0.1589 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 1.2035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 5289.6 | uss_MB: 4857.6 | child_rss_MB: 8120.4 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank6 [2026-10-27 19:56:23,053] INFO : Step 65000/100000 rank6 | Train Loss: 15.6652 | Train Loss Unweighted: 4.9833 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5112 | Regression_cos_phi: 0.0496 | Regression_energy: 0.6688 | Regression_eta: 0.0629 | Regression_pt: 0.6290 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6554 | physical_nopu: 4.4650 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3764 | Classification_binary: 3.6037 | Regression_cos_phi: 2.3673 | Regression_energy: 5.2423 | Regression_eta: 0.7968 | Regression_pt: 7.2999 | Regression_sin_phi: 2.2518 | LR: 1.38e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.7629s | Phase Times: max_pre_step_barrier: 0.1158 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0042 | max_backward: 0.1974 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.4176 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5282.6 | uss_MB: 4858.3 | child_rss_MB: 8025.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 124.9 +rank6 [2026-10-27 19:59:28,722] INFO : Step 66000/100000 rank6 | Train Loss: 12.0073 | Train Loss Unweighted: 4.3350 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4949 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3535 | Regression_eta: 0.0691 | Regression_pt: 0.3018 | Regression_sin_phi: 0.0501 | Group Train Loss Unweighted (window mean): physical_pu: 3.6526 | physical_nopu: 4.4657 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3364 | Classification_binary: 3.5123 | Regression_cos_phi: 1.8934 | Regression_energy: 1.7750 | Regression_eta: 0.6685 | Regression_pt: 2.4010 | Regression_sin_phi: 1.4329 | LR: 1.33e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.4470s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0891 | max_loss: 0.0538 | max_backward: 0.1468 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 1.2753 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5277.7 | uss_MB: 4858.9 | child_rss_MB: 7982.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 107.1 +rank6 [2026-10-27 20:02:51,910] INFO : Step 67000/100000 rank6 | Train Loss: 13.3177 | Train Loss Unweighted: 4.5554 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.4800 | Regression_cos_phi: 0.0500 | Regression_energy: 0.4690 | Regression_eta: 0.0673 | Regression_pt: 0.4253 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6538 | physical_nopu: 4.4665 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.5127 | Classification_binary: 3.5699 | Regression_cos_phi: 2.3189 | Regression_energy: 1.8226 | Regression_eta: 0.8301 | Regression_pt: 2.5360 | Regression_sin_phi: 1.7533 | LR: 1.28e-04 | DataLoad Time: 0.0015s | Model Forward Time: 3.9383s | Phase Times: max_pre_step_barrier: 0.0792 | max_batch_to_device: 0.0009 | max_forward: 0.0886 | max_loss: 0.1057 | max_backward: 0.1987 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 2.6109 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 5271.7 | uss_MB: 4859.8 | child_rss_MB: 7952.8 | allocated_MB: 393.9 | reserved_MB: 11792.0 | inactive_split_MB: 111.5 +rank6 [2026-10-27 20:06:09,103] INFO : Step 68000/100000 rank6 | Train Loss: 11.1535 | Train Loss Unweighted: 4.2643 | Train Loss Components: Classification: 0.0109 | Classification_binary: 3.5138 | Regression_cos_phi: 0.0456 | Regression_energy: 0.3163 | Regression_eta: 0.0648 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6527 | physical_nopu: 4.4693 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1979 | Classification_binary: 3.6383 | Regression_cos_phi: 1.8968 | Regression_energy: 1.2099 | Regression_eta: 0.9008 | Regression_pt: 1.3838 | Regression_sin_phi: 1.9485 | LR: 1.24e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.3899s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1008 | max_loss: 0.0767 | max_backward: 0.1768 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.1759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5263.4 | uss_MB: 4860.3 | child_rss_MB: 7883.7 | allocated_MB: 394.5 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank6 [2026-10-27 20:09:12,492] INFO : Step 69000/100000 rank6 | Train Loss: 12.1728 | Train Loss Unweighted: 4.3418 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4719 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3660 | Regression_eta: 0.0763 | Regression_pt: 0.3115 | Regression_sin_phi: 0.0548 | Group Train Loss Unweighted (window mean): physical_pu: 3.6494 | physical_nopu: 4.4629 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4243 | Classification_binary: 3.4526 | Regression_cos_phi: 1.6299 | Regression_energy: 1.5731 | Regression_eta: 0.7294 | Regression_pt: 1.8242 | Regression_sin_phi: 1.7647 | LR: 1.19e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.2956s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.1094 | max_loss: 0.0147 | max_backward: 0.1167 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 1.1204 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5251.4 | uss_MB: 4861.5 | child_rss_MB: 7816.8 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 106.5 +rank6 [2026-10-27 20:12:13,498] INFO : Step 70000/100000 rank6 | Train Loss: 14.9156 | Train Loss Unweighted: 4.7730 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.3975 | Regression_cos_phi: 0.0487 | Regression_energy: 0.6183 | Regression_eta: 0.0638 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0448 | Group Train Loss Unweighted (window mean): physical_pu: 3.6488 | physical_nopu: 4.4584 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3147 | Classification_binary: 3.2772 | Regression_cos_phi: 1.8091 | Regression_energy: 2.3621 | Regression_eta: 0.5707 | Regression_pt: 3.3434 | Regression_sin_phi: 1.4138 | LR: 1.14e-04 | DataLoad Time: 0.0068s | Model Forward Time: 2.3869s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1038 | max_loss: 0.0858 | max_backward: 0.1871 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 1.1619 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5245.8 | uss_MB: 4860.8 | child_rss_MB: 7775.8 | allocated_MB: 395.0 | reserved_MB: 11792.0 | inactive_split_MB: 111.9 +rank6 [2026-10-27 20:12:14,505] INFO : Running validation on rank6 +rank6 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 20:14:09,433] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 20:14:09,864] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 20:14:34,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 20:14:35,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 20:14:35,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 20:14:36,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 20:14:36,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 20:14:36,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 20:14:37,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 20:14:37,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 20:14:47,886] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 20:14:47,942] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 20:14:48,315] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 20:15:03,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 20:15:04,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 20:15:04,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 20:15:05,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 20:15:05,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 20:15:06,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 20:15:06,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 20:15:07,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 20:15:17,032] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 20:15:17,039] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 20:15:17,439] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 20:15:32,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 20:15:32,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 20:15:33,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 20:15:33,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 20:15:34,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 20:15:34,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 20:15:34,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 20:15:35,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 20:15:45,377] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-27 20:19:21,751] INFO : Step 71000/100000 rank6 | Train Loss: 10.9922 | Train Loss Unweighted: 3.6772 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9201 | Regression_cos_phi: 0.0477 | Regression_energy: 0.2906 | Regression_eta: 0.1066 | Regression_pt: 0.2485 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6484 | physical_nopu: 4.4561 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3953 | Classification_binary: 2.8377 | Regression_cos_phi: 1.7298 | Regression_energy: 1.4676 | Regression_eta: 1.2432 | Regression_pt: 1.8799 | Regression_sin_phi: 1.6906 | LR: 1.10e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.5364s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0652 | max_forward: 0.2596 | max_loss: 0.0036 | max_backward: 0.3497 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 1.8772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 7279.8 | uss_MB: 6900.2 | child_rss_MB: 7804.4 | allocated_MB: 405.9 | reserved_MB: 11130.0 | inactive_split_MB: 135.5 +rank6 [2026-10-27 20:22:34,778] INFO : Step 72000/100000 rank6 | Train Loss: 14.4733 | Train Loss Unweighted: 4.6898 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3667 | Regression_cos_phi: 0.0458 | Regression_energy: 0.5948 | Regression_eta: 0.0650 | Regression_pt: 0.5567 | Regression_sin_phi: 0.0444 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4536 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3690 | Classification_binary: 3.5106 | Regression_cos_phi: 1.6348 | Regression_energy: 3.1994 | Regression_eta: 0.5272 | Regression_pt: 4.4271 | Regression_sin_phi: 1.2782 | LR: 1.05e-04 | DataLoad Time: 0.0010s | Model Forward Time: 2.5079s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1390 | max_loss: 0.0037 | max_backward: 0.1007 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.2476 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1281 | Memory: rss_MB: 5316.2 | uss_MB: 4936.1 | child_rss_MB: 7727.0 | allocated_MB: 393.8 | reserved_MB: 11706.0 | inactive_split_MB: 88.5 +rank6 [2026-10-27 20:25:34,428] INFO : Step 73000/100000 rank6 | Train Loss: 11.7903 | Train Loss Unweighted: 4.2561 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.5010 | Regression_cos_phi: 0.0578 | Regression_energy: 0.3092 | Regression_eta: 0.0644 | Regression_pt: 0.2565 | Regression_sin_phi: 0.0557 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4567 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1073 | Classification_binary: 3.3235 | Regression_cos_phi: 1.8300 | Regression_energy: 1.2570 | Regression_eta: 0.8494 | Regression_pt: 1.2714 | Regression_sin_phi: 1.4917 | LR: 1.01e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.6061s | Phase Times: max_pre_step_barrier: 0.1246 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0122 | max_backward: 0.1972 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.2348 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5307.2 | uss_MB: 4937.3 | child_rss_MB: 7690.0 | allocated_MB: 393.5 | reserved_MB: 11706.0 | inactive_split_MB: 93.2 +rank6 [2026-10-27 20:28:47,370] INFO : Step 74000/100000 rank6 | Train Loss: 11.6748 | Train Loss Unweighted: 4.2803 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4309 | Regression_cos_phi: 0.0428 | Regression_energy: 0.3572 | Regression_eta: 0.0695 | Regression_pt: 0.3177 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.6485 | physical_nopu: 4.4502 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3962 | Classification_binary: 3.4434 | Regression_cos_phi: 1.7683 | Regression_energy: 1.7029 | Regression_eta: 0.6746 | Regression_pt: 2.2067 | Regression_sin_phi: 1.5105 | LR: 9.68e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.0486s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1195 | max_loss: 0.0060 | max_backward: 0.2082 | max_optimizer_step: 0.1763 | max_tensorboard_logging: 1.8023 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5300.9 | uss_MB: 4938.3 | child_rss_MB: 7614.5 | allocated_MB: 394.3 | reserved_MB: 11706.0 | inactive_split_MB: 97.0 +rank6 [2026-10-27 20:31:56,443] INFO : Step 75000/100000 rank6 | Train Loss: 14.1604 | Train Loss Unweighted: 4.6445 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.4007 | Regression_cos_phi: 0.0489 | Regression_energy: 0.5577 | Regression_eta: 0.0640 | Regression_pt: 0.5112 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.6489 | physical_nopu: 4.4484 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3302 | Classification_binary: 3.5461 | Regression_cos_phi: 2.0287 | Regression_energy: 2.2177 | Regression_eta: 0.7739 | Regression_pt: 2.8507 | Regression_sin_phi: 1.4434 | LR: 9.27e-05 | DataLoad Time: 0.0024s | Model Forward Time: 2.9328s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0831 | max_loss: 0.0038 | max_backward: 0.2058 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.5751 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1279 | Memory: rss_MB: 5290.9 | uss_MB: 4938.0 | child_rss_MB: 7581.3 | allocated_MB: 394.4 | reserved_MB: 11706.0 | inactive_split_MB: 89.1 +rank6 [2026-10-27 20:34:57,734] INFO : Step 76000/100000 rank6 | Train Loss: 11.4149 | Train Loss Unweighted: 4.2984 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5224 | Regression_cos_phi: 0.0478 | Regression_energy: 0.3298 | Regression_eta: 0.0709 | Regression_pt: 0.2685 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6454 | physical_nopu: 4.4486 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2978 | Classification_binary: 3.4922 | Regression_cos_phi: 1.7703 | Regression_energy: 1.4970 | Regression_eta: 0.6144 | Regression_pt: 1.7711 | Regression_sin_phi: 1.7583 | LR: 8.88e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.1733s | Phase Times: max_pre_step_barrier: 0.0296 | max_batch_to_device: 0.0996 | max_forward: 0.1816 | max_loss: 0.0036 | max_backward: 0.0930 | max_optimizer_step: 0.2829 | max_tensorboard_logging: 1.7104 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1287 | Memory: rss_MB: 5286.0 | uss_MB: 4937.3 | child_rss_MB: 7586.8 | allocated_MB: 393.6 | reserved_MB: 12330.0 | inactive_split_MB: 97.7 +rank6 [2026-10-27 20:38:16,884] INFO : Step 77000/100000 rank6 | Train Loss: 12.8789 | Train Loss Unweighted: 4.4653 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4333 | Regression_cos_phi: 0.0478 | Regression_energy: 0.4517 | Regression_eta: 0.0624 | Regression_pt: 0.4079 | Regression_sin_phi: 0.0461 | Group Train Loss Unweighted (window mean): physical_pu: 3.6433 | physical_nopu: 4.4448 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3882 | Classification_binary: 3.1033 | Regression_cos_phi: 1.2565 | Regression_energy: 2.1748 | Regression_eta: 0.3179 | Regression_pt: 2.9530 | Regression_sin_phi: 1.1129 | LR: 8.50e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.0364s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.0831 | max_loss: 0.0514 | max_backward: 0.1524 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.8652 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5273.0 | uss_MB: 4939.8 | child_rss_MB: 7463.9 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 99.2 +rank6 [2026-10-27 20:41:21,661] INFO : Step 78000/100000 rank6 | Train Loss: 13.2420 | Train Loss Unweighted: 4.6345 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.5815 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4598 | Regression_eta: 0.0679 | Regression_pt: 0.4144 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6415 | physical_nopu: 4.4494 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3241 | Classification_binary: 3.6407 | Regression_cos_phi: 2.2560 | Regression_energy: 1.7225 | Regression_eta: 0.8143 | Regression_pt: 2.0095 | Regression_sin_phi: 2.1323 | LR: 8.13e-05 | DataLoad Time: 0.0015s | Model Forward Time: 3.0723s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0927 | max_loss: 0.0036 | max_backward: 0.2060 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.8450 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5261.0 | uss_MB: 4940.8 | child_rss_MB: 7383.4 | allocated_MB: 394.7 | reserved_MB: 12330.0 | inactive_split_MB: 95.1 +rank6 [2026-10-27 20:44:31,405] INFO : Step 79000/100000 rank6 | Train Loss: 11.5047 | Train Loss Unweighted: 4.2761 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4919 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3261 | Regression_eta: 0.0640 | Regression_pt: 0.2834 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6416 | physical_nopu: 4.4460 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2956 | Classification_binary: 3.5487 | Regression_cos_phi: 1.6818 | Regression_energy: 1.5083 | Regression_eta: 0.8147 | Regression_pt: 1.8620 | Regression_sin_phi: 1.8164 | LR: 7.78e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.2159s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0253 | max_forward: 0.1473 | max_loss: 0.0038 | max_backward: 0.0955 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 1.9569 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5248.8 | uss_MB: 4513.6 | child_rss_MB: 7321.5 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 120.0 +rank6 [2026-10-27 20:47:43,818] INFO : Step 80000/100000 rank6 | Train Loss: 15.3065 | Train Loss Unweighted: 4.8564 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4208 | Regression_cos_phi: 0.0455 | Regression_energy: 0.6415 | Regression_eta: 0.0773 | Regression_pt: 0.6092 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6398 | physical_nopu: 4.4406 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3370 | Classification_binary: 3.2599 | Regression_cos_phi: 1.6139 | Regression_energy: 3.2474 | Regression_eta: 0.5905 | Regression_pt: 4.6047 | Regression_sin_phi: 1.6229 | LR: 7.44e-05 | DataLoad Time: 0.0069s | Model Forward Time: 3.0655s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0010 | max_forward: 0.0840 | max_loss: 0.0039 | max_backward: 0.0916 | max_optimizer_step: 0.2630 | max_tensorboard_logging: 1.7339 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5238.8 | uss_MB: 4942.0 | child_rss_MB: 7231.1 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 141.7 +rank6 [2026-10-27 20:47:44,748] INFO : Running validation on rank6 +rank6 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 20:49:38,529] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 20:49:38,981] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 20:50:04,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 20:50:05,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 20:50:05,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 20:50:06,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 20:50:06,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 20:50:07,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 20:50:07,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 20:50:08,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 20:50:18,235] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 20:50:18,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 20:50:18,743] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 20:50:33,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 20:50:33,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 20:50:34,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 20:50:34,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 20:50:35,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 20:50:35,827] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 20:50:36,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 20:50:36,728] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 20:50:46,739] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 20:50:47,267] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 20:50:47,676] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 20:51:02,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 20:51:03,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 20:51:03,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 20:51:04,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 20:51:04,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 20:51:05,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 20:51:05,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 20:51:05,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 20:51:15,904] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 20:54:47,405] INFO : Step 81000/100000 rank6 | Train Loss: 10.7072 | Train Loss Unweighted: 3.6241 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.9080 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2710 | Regression_eta: 0.1078 | Regression_pt: 0.2269 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6413 | physical_nopu: 4.4395 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3738 | Classification_binary: 2.9187 | Regression_cos_phi: 1.6699 | Regression_energy: 1.2875 | Regression_eta: 1.1672 | Regression_pt: 1.5735 | Regression_sin_phi: 1.5670 | LR: 7.11e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.4726s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0033 | max_forward: 0.1984 | max_loss: 0.0038 | max_backward: 0.3244 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 1.8991 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 7314.5 | uss_MB: 7027.3 | child_rss_MB: 7215.1 | allocated_MB: 408.0 | reserved_MB: 11650.0 | inactive_split_MB: 261.6 +rank6 [2026-10-27 20:58:04,710] INFO : Step 82000/100000 rank6 | Train Loss: 12.9094 | Train Loss Unweighted: 4.6043 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5247 | Regression_cos_phi: 0.0417 | Regression_energy: 0.4765 | Regression_eta: 0.0603 | Regression_pt: 0.4449 | Regression_sin_phi: 0.0402 | Group Train Loss Unweighted (window mean): physical_pu: 3.6409 | physical_nopu: 4.4420 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4378 | Classification_binary: 3.4121 | Regression_cos_phi: 1.4114 | Regression_energy: 2.9515 | Regression_eta: 0.4752 | Regression_pt: 4.1592 | Regression_sin_phi: 1.0535 | LR: 6.80e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.1711s | Phase Times: max_pre_step_barrier: 0.0156 | max_batch_to_device: 0.0008 | max_forward: 0.1472 | max_loss: 0.0038 | max_backward: 0.1553 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 1.8498 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5342.3 | uss_MB: 5059.9 | child_rss_MB: 7159.0 | allocated_MB: 393.8 | reserved_MB: 11650.0 | inactive_split_MB: 92.9 +rank6 [2026-10-27 21:01:20,554] INFO : Step 83000/100000 rank6 | Train Loss: 11.2556 | Train Loss Unweighted: 4.2618 | Train Loss Components: Classification: 0.0094 | Classification_binary: 3.5167 | Regression_cos_phi: 0.0501 | Regression_energy: 0.3127 | Regression_eta: 0.0669 | Regression_pt: 0.2586 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6408 | physical_nopu: 4.4397 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2087 | Classification_binary: 3.6249 | Regression_cos_phi: 2.0704 | Regression_energy: 1.6304 | Regression_eta: 0.8184 | Regression_pt: 2.0400 | Regression_sin_phi: 1.8228 | LR: 6.51e-05 | DataLoad Time: 0.0018s | Model Forward Time: 2.5752s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0008 | max_forward: 0.1683 | max_loss: 0.0037 | max_backward: 0.1023 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.3519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 5336.3 | uss_MB: 5061.6 | child_rss_MB: 7095.7 | allocated_MB: 393.2 | reserved_MB: 11650.0 | inactive_split_MB: 95.2 +rank6 [2026-10-27 21:04:33,664] INFO : Step 84000/100000 rank6 | Train Loss: 11.5964 | Train Loss Unweighted: 4.2611 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4346 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3486 | Regression_eta: 0.0638 | Regression_pt: 0.3055 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6366 | physical_nopu: 4.4419 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4488 | Classification_binary: 3.3670 | Regression_cos_phi: 1.5552 | Regression_energy: 1.7929 | Regression_eta: 0.7725 | Regression_pt: 2.3470 | Regression_sin_phi: 1.5565 | LR: 6.23e-05 | DataLoad Time: 0.2919s | Model Forward Time: 4.6438s | Phase Times: max_pre_step_barrier: 0.2933 | max_batch_to_device: 0.0008 | max_forward: 0.1129 | max_loss: 0.0110 | max_backward: 0.1312 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 3.4426 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5330.3 | uss_MB: 5063.1 | child_rss_MB: 7029.9 | allocated_MB: 393.9 | reserved_MB: 11650.0 | inactive_split_MB: 95.6 +rank6 [2026-10-27 21:07:28,408] INFO : Step 85000/100000 rank6 | Train Loss: 14.1090 | Train Loss Unweighted: 4.6641 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.3775 | Regression_cos_phi: 0.0425 | Regression_energy: 0.5754 | Regression_eta: 0.0615 | Regression_pt: 0.5479 | Regression_sin_phi: 0.0420 | Group Train Loss Unweighted (window mean): physical_pu: 3.6325 | physical_nopu: 4.4336 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3842 | Classification_binary: 3.4613 | Regression_cos_phi: 2.4724 | Regression_energy: 3.0175 | Regression_eta: 0.8219 | Regression_pt: 4.2501 | Regression_sin_phi: 1.5876 | LR: 5.96e-05 | DataLoad Time: 0.0010s | Model Forward Time: 2.5402s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0496 | max_backward: 0.1488 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 1.3583 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 5323.1 | uss_MB: 5063.4 | child_rss_MB: 7037.3 | allocated_MB: 394.0 | reserved_MB: 11698.0 | inactive_split_MB: 93.1 +rank6 [2026-10-27 21:10:26,799] INFO : Step 86000/100000 rank6 | Train Loss: 11.7921 | Train Loss Unweighted: 4.3412 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.5823 | Regression_cos_phi: 0.0537 | Regression_energy: 0.3118 | Regression_eta: 0.0750 | Regression_pt: 0.2521 | Regression_sin_phi: 0.0550 | Group Train Loss Unweighted (window mean): physical_pu: 3.6404 | physical_nopu: 4.4337 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2334 | Classification_binary: 3.6286 | Regression_cos_phi: 1.7930 | Regression_energy: 1.5536 | Regression_eta: 1.2839 | Regression_pt: 1.7726 | Regression_sin_phi: 1.9064 | LR: 5.71e-05 | DataLoad Time: 0.0018s | Model Forward Time: 2.4343s | Phase Times: max_pre_step_barrier: 0.0233 | max_batch_to_device: 0.0898 | max_forward: 0.1824 | max_loss: 0.0038 | max_backward: 0.0938 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.9995 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1323 | Memory: rss_MB: 5319.1 | uss_MB: 4616.7 | child_rss_MB: 7006.7 | allocated_MB: 393.2 | reserved_MB: 11704.0 | inactive_split_MB: 96.0 +rank6 [2026-10-27 21:13:31,178] INFO : Step 87000/100000 rank6 | Train Loss: 12.0054 | Train Loss Unweighted: 4.3232 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.3964 | Regression_cos_phi: 0.0444 | Regression_energy: 0.4046 | Regression_eta: 0.0620 | Regression_pt: 0.3567 | Regression_sin_phi: 0.0418 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4064 | Classification_binary: 3.4208 | Regression_cos_phi: 1.5753 | Regression_energy: 1.6958 | Regression_eta: 0.4883 | Regression_pt: 2.1949 | Regression_sin_phi: 1.3862 | LR: 5.48e-05 | DataLoad Time: 0.0021s | Model Forward Time: 2.3323s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.1290 | max_loss: 0.0039 | max_backward: 0.0975 | max_optimizer_step: 0.1784 | max_tensorboard_logging: 1.1620 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5310.9 | uss_MB: 5064.7 | child_rss_MB: 7011.5 | allocated_MB: 394.7 | reserved_MB: 11704.0 | inactive_split_MB: 94.3 +rank6 [2026-10-27 21:16:33,434] INFO : Step 88000/100000 rank6 | Train Loss: 14.2663 | Train Loss Unweighted: 4.6673 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.4533 | Regression_cos_phi: 0.0523 | Regression_energy: 0.5342 | Regression_eta: 0.0666 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0508 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2934 | Classification_binary: 3.4169 | Regression_cos_phi: 1.8199 | Regression_energy: 3.3969 | Regression_eta: 0.7783 | Regression_pt: 4.7934 | Regression_sin_phi: 1.6867 | LR: 5.26e-05 | DataLoad Time: 0.0025s | Model Forward Time: 2.3216s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0008 | max_forward: 0.0997 | max_loss: 0.0039 | max_backward: 0.1038 | max_optimizer_step: 0.2517 | max_tensorboard_logging: 1.0175 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5309.9 | uss_MB: 5065.2 | child_rss_MB: 7012.4 | allocated_MB: 393.7 | reserved_MB: 11704.0 | inactive_split_MB: 103.2 +rank6 [2026-10-27 21:19:30,808] INFO : Step 89000/100000 rank6 | Train Loss: 11.9885 | Train Loss Unweighted: 4.3122 | Train Loss Components: Classification: 0.0138 | Classification_binary: 3.4837 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3438 | Regression_eta: 0.0687 | Regression_pt: 0.2984 | Regression_sin_phi: 0.0554 | Group Train Loss Unweighted (window mean): physical_pu: 3.6329 | physical_nopu: 4.4374 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2846 | Classification_binary: 3.4252 | Regression_cos_phi: 1.5523 | Regression_energy: 1.3184 | Regression_eta: 0.5782 | Regression_pt: 1.7712 | Regression_sin_phi: 1.6586 | LR: 5.06e-05 | DataLoad Time: 0.0015s | Model Forward Time: 2.2341s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0976 | max_loss: 0.0053 | max_backward: 0.1242 | max_optimizer_step: 0.1785 | max_tensorboard_logging: 1.0326 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 5307.5 | uss_MB: 5066.4 | child_rss_MB: 7006.7 | allocated_MB: 395.0 | reserved_MB: 13422.0 | inactive_split_MB: 109.9 +rank6 [2026-10-27 21:22:22,522] INFO : Step 90000/100000 rank6 | Train Loss: 10.5888 | Train Loss Unweighted: 3.6256 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9282 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2656 | Regression_eta: 0.1044 | Regression_pt: 0.2178 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6354 | physical_nopu: 4.4354 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3458 | Classification_binary: 2.8840 | Regression_cos_phi: 1.7018 | Regression_energy: 1.3105 | Regression_eta: 1.2519 | Regression_pt: 1.5617 | Regression_sin_phi: 1.5485 | LR: 4.88e-05 | DataLoad Time: 0.0020s | Model Forward Time: 4.1185s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0033 | max_forward: 0.1953 | max_loss: 0.0645 | max_backward: 0.8258 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.9975 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5307.5 | uss_MB: 5074.7 | child_rss_MB: 7021.2 | allocated_MB: 404.6 | reserved_MB: 13422.0 | inactive_split_MB: 208.4 +rank6 [2026-10-27 21:22:23,524] INFO : Running validation on rank6 +rank6 [2026-10-27 21:22:23,524] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 21:22:23,524] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 21:22:23,524] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 21:22:23,524] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 21:24:23,625] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 21:24:24,116] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 21:24:48,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 21:24:49,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 21:24:49,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 21:24:50,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 21:24:50,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 21:24:51,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 21:24:51,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 21:24:52,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 21:25:02,155] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 21:25:02,472] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 21:25:02,869] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 21:25:18,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 21:25:18,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 21:25:19,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 21:25:19,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 21:25:20,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 21:25:20,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 21:25:21,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 21:25:21,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 21:25:31,470] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 21:25:32,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 21:25:32,493] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 21:25:47,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 21:25:48,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 21:25:48,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 21:25:48,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 21:25:49,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 21:25:50,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 21:25:50,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 21:25:50,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 21:26:00,996] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 21:29:48,334] INFO : Step 91000/100000 rank6 | Train Loss: 11.9808 | Train Loss Unweighted: 4.3520 | Train Loss Components: Classification: 0.0128 | Classification_binary: 3.5260 | Regression_cos_phi: 0.0522 | Regression_energy: 0.3480 | Regression_eta: 0.0723 | Regression_pt: 0.2901 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4346 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2631 | Classification_binary: 3.3203 | Regression_cos_phi: 1.2863 | Regression_energy: 1.6416 | Regression_eta: 0.4374 | Regression_pt: 2.2532 | Regression_sin_phi: 1.3651 | LR: 4.71e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.6016s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1743 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.3267 | max_tensorboard_logging: 2.0733 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0716 | Memory: rss_MB: 7308.2 | uss_MB: 7066.1 | child_rss_MB: 7044.9 | allocated_MB: 393.7 | reserved_MB: 11392.0 | inactive_split_MB: 89.1 +rank6 [2026-10-27 21:33:08,656] INFO : Step 92000/100000 rank6 | Train Loss: 14.4974 | Train Loss Unweighted: 4.7629 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4460 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5917 | Regression_eta: 0.0739 | Regression_pt: 0.5467 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6297 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3021 | Classification_binary: 3.4420 | Regression_cos_phi: 1.5262 | Regression_energy: 3.3931 | Regression_eta: 0.7860 | Regression_pt: 4.5896 | Regression_sin_phi: 1.1955 | LR: 4.57e-05 | DataLoad Time: 0.0020s | Model Forward Time: 2.6195s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1620 | max_loss: 0.0037 | max_backward: 0.2478 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 1.1676 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0595 | Memory: rss_MB: 7302.2 | uss_MB: 6481.6 | child_rss_MB: 6997.3 | allocated_MB: 394.0 | reserved_MB: 11392.0 | inactive_split_MB: 90.6 +rank6 [2026-10-27 21:36:11,295] INFO : Step 93000/100000 rank6 | Train Loss: 13.1041 | Train Loss Unweighted: 4.5189 | Train Loss Components: Classification: 0.0119 | Classification_binary: 3.5265 | Regression_cos_phi: 0.0575 | Regression_energy: 0.4342 | Regression_eta: 0.0649 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6346 | physical_nopu: 4.4327 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2808 | Classification_binary: 3.7815 | Regression_cos_phi: 2.0492 | Regression_energy: 2.0050 | Regression_eta: 0.8282 | Regression_pt: 2.4143 | Regression_sin_phi: 1.7816 | LR: 4.43e-05 | DataLoad Time: 0.0021s | Model Forward Time: 2.4220s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1013 | max_loss: 0.1069 | max_backward: 0.2035 | max_optimizer_step: 0.1792 | max_tensorboard_logging: 1.1596 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 5379.1 | uss_MB: 5141.2 | child_rss_MB: 6951.6 | allocated_MB: 393.6 | reserved_MB: 13294.0 | inactive_split_MB: 89.2 +rank6 [2026-10-27 21:39:11,302] INFO : Step 94000/100000 rank6 | Train Loss: 11.5177 | Train Loss Unweighted: 4.2756 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4441 | Regression_cos_phi: 0.0413 | Regression_energy: 0.3503 | Regression_eta: 0.0686 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6308 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4197 | Classification_binary: 3.5010 | Regression_cos_phi: 1.6457 | Regression_energy: 1.6015 | Regression_eta: 0.7978 | Regression_pt: 2.0123 | Regression_sin_phi: 1.9574 | LR: 4.32e-05 | DataLoad Time: 0.0019s | Model Forward Time: 6.3165s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0191 | max_forward: 0.1721 | max_loss: 0.0188 | max_backward: 0.2028 | max_optimizer_step: 0.2895 | max_tensorboard_logging: 4.7123 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 5372.1 | uss_MB: 5157.2 | child_rss_MB: 6925.7 | allocated_MB: 394.1 | reserved_MB: 13294.0 | inactive_split_MB: 89.1 +rank6 [2026-10-27 21:42:20,694] INFO : Step 95000/100000 rank6 | Train Loss: 14.1207 | Train Loss Unweighted: 4.6477 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3920 | Regression_cos_phi: 0.0441 | Regression_energy: 0.5776 | Regression_eta: 0.0602 | Regression_pt: 0.5087 | Regression_sin_phi: 0.0499 | Group Train Loss Unweighted (window mean): physical_pu: 3.6267 | physical_nopu: 4.4321 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3360 | Classification_binary: 3.4004 | Regression_cos_phi: 1.2763 | Regression_energy: 2.1572 | Regression_eta: 0.5558 | Regression_pt: 2.9401 | Regression_sin_phi: 1.0845 | LR: 4.22e-05 | DataLoad Time: 0.0016s | Model Forward Time: 2.5038s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.0941 | max_loss: 0.0036 | max_backward: 0.1734 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 1.2811 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5361.0 | uss_MB: 5157.8 | child_rss_MB: 6928.3 | allocated_MB: 396.0 | reserved_MB: 14294.0 | inactive_split_MB: 91.1 +rank6 [2026-10-27 21:45:16,251] INFO : Step 96000/100000 rank6 | Train Loss: 11.5036 | Train Loss Unweighted: 4.2276 | Train Loss Components: Classification: 0.0097 | Classification_binary: 3.4897 | Regression_cos_phi: 0.0538 | Regression_energy: 0.2982 | Regression_eta: 0.0802 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6278 | physical_nopu: 4.4329 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2083 | Classification_binary: 3.6814 | Regression_cos_phi: 2.3850 | Regression_energy: 2.1319 | Regression_eta: 1.0848 | Regression_pt: 2.5931 | Regression_sin_phi: 1.9167 | LR: 4.14e-05 | DataLoad Time: 0.0018s | Model Forward Time: 2.3873s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0007 | max_forward: 0.0991 | max_loss: 0.0037 | max_backward: 0.1000 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5357.0 | uss_MB: 5158.4 | child_rss_MB: 6899.7 | allocated_MB: 393.0 | reserved_MB: 14294.0 | inactive_split_MB: 95.3 +rank6 [2026-10-27 21:48:34,573] INFO : Step 97000/100000 rank6 | Train Loss: 11.8793 | Train Loss Unweighted: 4.3044 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.4420 | Regression_cos_phi: 0.0454 | Regression_energy: 0.3663 | Regression_eta: 0.0711 | Regression_pt: 0.3156 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.6277 | physical_nopu: 4.4255 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3852 | Classification_binary: 3.4705 | Regression_cos_phi: 1.6328 | Regression_energy: 1.9626 | Regression_eta: 0.9005 | Regression_pt: 2.2393 | Regression_sin_phi: 1.6361 | LR: 4.08e-05 | DataLoad Time: 0.0016s | Model Forward Time: 2.5917s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1192 | max_loss: 0.0048 | max_backward: 0.1852 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 5356.0 | uss_MB: 5157.9 | child_rss_MB: 6903.2 | allocated_MB: 394.7 | reserved_MB: 14294.0 | inactive_split_MB: 98.1 +rank6 [2026-10-27 21:51:32,851] INFO : Step 98000/100000 rank6 | Train Loss: 12.9367 | Train Loss Unweighted: 4.4840 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0441 | Regression_energy: 0.4811 | Regression_eta: 0.0611 | Regression_pt: 0.4378 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3667 | Classification_binary: 3.3450 | Regression_cos_phi: 1.6230 | Regression_energy: 1.8400 | Regression_eta: 0.6218 | Regression_pt: 2.3409 | Regression_sin_phi: 1.4908 | LR: 4.04e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.8943s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1318 | max_loss: 0.0038 | max_backward: 0.1127 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.6868 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 5362.0 | uss_MB: 5160.3 | child_rss_MB: 6899.7 | allocated_MB: 394.5 | reserved_MB: 14294.0 | inactive_split_MB: 82.0 +rank6 [2026-10-27 21:54:35,739] INFO : Step 99000/100000 rank6 | Train Loss: 11.6408 | Train Loss Unweighted: 4.3204 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.5557 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3117 | Regression_eta: 0.0641 | Regression_pt: 0.2727 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.6292 | physical_nopu: 4.4284 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3254 | Classification_binary: 3.6229 | Regression_cos_phi: 1.7248 | Regression_energy: 1.9426 | Regression_eta: 0.7741 | Regression_pt: 2.5421 | Regression_sin_phi: 2.4567 | LR: 4.01e-05 | DataLoad Time: 0.0083s | Model Forward Time: 3.0806s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1731 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.8331 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 5358.0 | uss_MB: 5160.5 | child_rss_MB: 6886.0 | allocated_MB: 394.3 | reserved_MB: 14294.0 | inactive_split_MB: 113.7 +rank6 [2026-10-27 21:57:56,509] INFO : Step 100000/100000 rank6 | Train Loss: 10.8909 | Train Loss Unweighted: 3.6545 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9133 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2833 | Regression_eta: 0.1041 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6336 | physical_nopu: 4.4311 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3530 | Classification_binary: 2.8583 | Regression_cos_phi: 1.7563 | Regression_energy: 1.3815 | Regression_eta: 1.1312 | Regression_pt: 1.7418 | Regression_sin_phi: 1.6072 | LR: 4.00e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.7457s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0030 | max_forward: 0.2088 | max_loss: 0.0393 | max_backward: 0.3031 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.9276 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 5350.9 | uss_MB: 5163.3 | child_rss_MB: 6834.2 | allocated_MB: 405.7 | reserved_MB: 14294.0 | inactive_split_MB: 165.9 +rank6 [2026-10-27 21:57:57,461] INFO : Running validation on rank6 +rank6 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 21:59:51,633] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 21:59:52,049] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 22:00:18,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 22:00:19,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 22:00:19,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 22:00:20,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 22:00:20,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 22:00:20,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 22:00:21,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 22:00:21,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 22:00:31,879] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 22:00:32,062] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 22:00:32,464] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 22:00:47,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 22:00:48,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 22:00:48,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 22:00:49,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 22:00:49,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 22:00:50,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 22:00:50,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 22:00:51,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 22:01:01,095] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 22:01:01,522] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 22:01:01,928] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 22:01:16,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 22:01:17,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 22:01:17,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 22:01:18,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 22:01:18,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 22:01:19,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 22:01:19,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 22:01:19,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 22:01:29,937] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-27 22:01:31,053] INFO : Training completed. Total time on device 6: 350.233min diff --git a/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.7 b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.7 new file mode 100644 index 0000000000000000000000000000000000000000..0167d18cfa467388704351f1d583b0b4c6c4f6ec --- /dev/null +++ b/cms/v3.2.0/attention_seed12345_20261027_160940_955225/train.log.7 @@ -0,0 +1,851 @@ +rank7 [2026-10-27 16:10:10,455] INFO : Initializing model with seed=12345; process seed=12352 +rank7 [2026-10-27 16:10:10,456] INFO : configured dtype=torch.bfloat16 for autocast +rank7 [2026-10-27 16:10:37,169] INFO : Instantiating model +rank7 [2026-10-27 16:10:37,169] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank7 [2026-10-27 16:10:37,169] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank7 [2026-10-27 16:10:37,200] INFO : Input encoding initialization took 0.03s +rank7 [2026-10-27 16:10:37,200] INFO : nn0 parameters: 2130944 +rank7 [2026-10-27 16:10:37,201] INFO : Initializing shared backbone layers, num_convs=3 +rank7 [2026-10-27 16:10:37,201] INFO : Initializing backbone layer 0 +rank7 [2026-10-27 16:10:37,201] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-27 16:10:37,211] INFO : Initializing backbone layer 1 +rank7 [2026-10-27 16:10:37,212] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-27 16:10:37,222] INFO : Initializing backbone layer 2 +rank7 [2026-10-27 16:10:37,222] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-27 16:10:37,232] INFO : Backbone initialization took 0.03s +rank7 [2026-10-27 16:10:37,232] INFO : backbone parameters: 4733952 +rank7 [2026-10-27 16:10:37,232] INFO : Initializing output DNNs +rank7 [2026-10-27 16:10:37,266] INFO : Output DNNs initialization took 0.03s +rank7 [2026-10-27 16:10:37,266] INFO : backbone_mode=BackboneMode.SHARED +rank7 [2026-10-27 16:10:37,266] INFO : jagged_attention enabled=True +rank7 [2026-10-27 16:10:37,266] INFO : flash_attn_varlen enabled=True +rank7 [2026-10-27 16:10:37,266] INFO : task_queries enabled=False +rank7 [2026-10-27 16:10:37,266] INFO : classification_norm parameters: 1024 +rank7 [2026-10-27 16:10:37,266] INFO : regression_norm parameters: 1024 +rank7 [2026-10-27 16:10:37,266] INFO : classification_query parameters: 0 +rank7 [2026-10-27 16:10:37,266] INFO : regression_query parameters: 0 +rank7 [2026-10-27 16:10:37,266] INFO : classification_readout parameters: 0 +rank7 [2026-10-27 16:10:37,266] INFO : regression_readout parameters: 0 +rank7 [2026-10-27 16:10:37,266] INFO : output_mode=elementwise +rank7 [2026-10-27 16:10:37,266] INFO : set_decoder parameters: 0 +rank7 [2026-10-27 16:10:37,266] INFO : nn_binary_particle parameters: 264706 +rank7 [2026-10-27 16:10:37,266] INFO : nn_pid parameters: 268297 +rank7 [2026-10-27 16:10:37,266] INFO : nn_pt parameters: 2113544 +rank7 [2026-10-27 16:10:37,267] INFO : nn_eta parameters: 264706 +rank7 [2026-10-27 16:10:37,267] INFO : nn_sin_phi parameters: 264706 +rank7 [2026-10-27 16:10:37,267] INFO : nn_cos_phi parameters: 264706 +rank7 [2026-10-27 16:10:37,267] INFO : nn_energy parameters: 2113544 +rank7 [2026-10-27 16:10:37,267] INFO : Total MLPF parameters: 12421153 +rank7 [2026-10-27 16:10:37,267] INFO : MLPF __init__ done +rank7 [2026-10-27 16:10:37,270] INFO : Instantiated model +rank7 [2026-10-27 16:10:37,270] INFO : Moving model to device rank=7 +rank7 [2026-10-27 16:10:37,300] INFO : Moved model to device rank=7 +rank7 [2026-10-27 16:10:37,301] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank7 [2026-10-27 16:10:37,301] INFO : Configured model for SyncBatchNorm rank=7 +rank7 [2026-10-27 16:10:37,493] INFO : Configured model for DistributedDataParallel rank=7 +rank7 [2026-10-27 16:10:37,595] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +167 module.nn_energy.nn.7.0.bias ... 0 +168 module.nn_energy.nn.7.2.weight ... 0 +169 module.nn_energy.nn.7.2.bias ... 0 +170 module.nn_energy.nn.7.4.weight ... 0 +171 module.nn_energy.nn.7.4.bias ... 0 + +[172 rows x 3 columns] +rank7 [2026-10-27 16:10:37,595] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=512, out_features=512, bias=True) + ) + (norm0): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=512, out_features=512, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank7 [2026-10-27 16:10:37,596] INFO : Trainable parameters: 12421153 +rank7 [2026-10-27 16:10:37,596] INFO : Non-trainable parameters: 0 +rank7 [2026-10-27 16:10:37,596] INFO : Total parameters: 12421153 +rank7 [2026-10-27 16:10:37,599] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.mha.in_proj_weight 786432 0 + module.backbone.0.mha.in_proj_bias 1536 0 +module.backbone.0.mha.out_proj.weight 262144 0 + module.backbone.0.mha.out_proj.bias 512 0 + module.backbone.0.norm0.weight 512 0 + module.backbone.0.norm0.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm1.bias 512 0 + module.backbone.0.seq.0.weight 262144 0 + module.backbone.0.seq.0.bias 512 0 + module.backbone.0.seq.2.weight 262144 0 + module.backbone.0.seq.2.bias 512 0 + module.backbone.1.mha.in_proj_weight 786432 0 + module.backbone.1.mha.in_proj_bias 1536 0 +module.backbone.1.mha.out_proj.weight 262144 0 + module.backbone.1.mha.out_proj.bias 512 0 + module.backbone.1.norm0.weight 512 0 + module.backbone.1.norm0.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm1.bias 512 0 + module.backbone.1.seq.0.weight 262144 0 + module.backbone.1.seq.0.bias 512 0 + module.backbone.1.seq.2.weight 262144 0 + module.backbone.1.seq.2.bias 512 0 + module.backbone.2.mha.in_proj_weight 786432 0 + module.backbone.2.mha.in_proj_bias 1536 0 +module.backbone.2.mha.out_proj.weight 262144 0 + module.backbone.2.mha.out_proj.bias 512 0 + module.backbone.2.norm0.weight 512 0 + module.backbone.2.norm0.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm1.bias 512 0 + module.backbone.2.seq.0.weight 262144 0 + module.backbone.2.seq.0.bias 512 0 + module.backbone.2.seq.2.weight 262144 0 + module.backbone.2.seq.2.bias 512 0 + module.classification_norm.weight 512 0 + module.classification_norm.bias 512 0 + module.regression_norm.weight 512 0 + module.regression_norm.bias 512 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank7 [2026-10-27 16:10:37,599] INFO : Getting dataloaders +rank7 [2026-10-27 16:10:37,599] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:43,312] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:44,040] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:44,837] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank7 [2026-10-27 16:10:44,838] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:46,094] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:46,993] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:48,057] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank7 [2026-10-27 16:10:48,057] INFO : Creating WeightedMixIterator with 2 data loaders. +rank7 [2026-10-27 16:10:48,058] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:48,225] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:48,373] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:48,516] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank7 [2026-10-27 16:10:48,516] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:48,683] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:48,840] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:48,983] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank7 [2026-10-27 16:10:48,983] INFO : Creating InterleavedIterator with 2 data loaders. +rank7 [2026-10-27 16:11:16,695] INFO : Got dataloaders +rank7 [2026-10-27 16:11:16,695] INFO : loader split=train rank=7 len=232058 +rank7 [2026-10-27 16:11:16,695] INFO : loader split=valid rank=7 len=48 +rank7 [2026-10-27 16:11:16,696] INFO : Starting training from step 1 to 100000 on rank 7 of 8 +rank7 [2026-10-27 16:11:16,696] INFO : Creating train_iterator +rank7 [2026-10-27 16:11:16,696] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank7 [2026-10-27 16:11:30,664] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank7 [2026-10-27 16:11:45,454] INFO : Created train_iterator +rank7 [2026-10-27 16:12:23,642] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.123548031449318, 'Classification': 0.2298217851272784, 'Regression_pt': 0.7194055180624127, 'Regression_eta': 0.4619348940253258, 'Regression_sin_phi': 0.15005176125559955, 'Regression_cos_phi': 0.14165307599119842, 'Regression_energy': 1.0569261094555258}, weights={'Classification_binary': 1.0, 'Classification': 22.293569564819336, 'Regression_pt': 7.12191915512085, 'Regression_eta': 11.091493606567383, 'Regression_sin_phi': 34.14520263671875, 'Regression_cos_phi': 36.16969299316406, 'Regression_energy': 4.847593307495117} +rank7 [2026-10-27 16:14:48,546] INFO : Step 1000/100000 rank7 | Train Loss: 15.9738 | Train Loss Unweighted: 5.5021 | Train Loss Components: Classification: 0.0191 | Classification_binary: 4.1436 | Regression_cos_phi: 0.0516 | Regression_energy: 0.6035 | Regression_eta: 0.0819 | Regression_pt: 0.5500 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 5.2846 | physical_nopu: 6.1663 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4128 | Classification_binary: 4.0209 | Regression_cos_phi: 2.8331 | Regression_energy: 2.1509 | Regression_eta: 0.7901 | Regression_pt: 2.7659 | Regression_sin_phi: 1.5775 | LR: 4.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.9578s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0013 | max_forward: 0.0194 | max_loss: 0.0039 | max_backward: 0.0308 | max_optimizer_step: 0.1391 | max_tensorboard_logging: 1.0816 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0030 | Memory: rss_MB: 4094.8 | uss_MB: 2885.6 | child_rss_MB: 10815.8 | allocated_MB: 395.5 | reserved_MB: 11198.0 | inactive_split_MB: 104.9 +rank7 [2026-10-27 16:17:31,218] INFO : Step 2000/100000 rank7 | Train Loss: 17.2752 | Train Loss Unweighted: 5.6831 | Train Loss Components: Classification: 0.0157 | Classification_binary: 4.0887 | Regression_cos_phi: 0.0506 | Regression_energy: 0.7236 | Regression_eta: 0.0696 | Regression_pt: 0.6798 | Regression_sin_phi: 0.0552 | Group Train Loss Unweighted (window mean): physical_pu: 4.4021 | physical_nopu: 5.4797 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3451 | Classification_binary: 4.0477 | Regression_cos_phi: 1.7607 | Regression_energy: 5.0480 | Regression_eta: 0.7509 | Regression_pt: 7.1943 | Regression_sin_phi: 1.3698 | LR: 4.00e-04 | DataLoad Time: 0.0009s | Model Forward Time: 1.7803s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0250 | max_loss: 0.0036 | max_backward: 0.0657 | max_optimizer_step: 0.1397 | max_tensorboard_logging: 0.8949 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4097.7 | uss_MB: 2895.6 | child_rss_MB: 10834.7 | allocated_MB: 394.2 | reserved_MB: 11198.0 | inactive_split_MB: 90.1 +rank7 [2026-10-27 16:20:19,986] INFO : Step 3000/100000 rank7 | Train Loss: 14.0295 | Train Loss Unweighted: 5.1258 | Train Loss Components: Classification: 0.0148 | Classification_binary: 4.0405 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4749 | Regression_eta: 0.0746 | Regression_pt: 0.4205 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 4.1913 | physical_nopu: 5.3261 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3410 | Classification_binary: 3.9849 | Regression_cos_phi: 1.9181 | Regression_energy: 2.1589 | Regression_eta: 0.9769 | Regression_pt: 2.8522 | Regression_sin_phi: 2.1284 | LR: 3.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.4932s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1067 | max_loss: 0.0036 | max_backward: 0.0365 | max_optimizer_step: 0.1415 | max_tensorboard_logging: 1.1098 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0093 | Memory: rss_MB: 4102.6 | uss_MB: 2907.4 | child_rss_MB: 10837.4 | allocated_MB: 395.4 | reserved_MB: 11784.0 | inactive_split_MB: 88.5 +rank7 [2026-10-27 16:23:08,050] INFO : Step 4000/100000 rank7 | Train Loss: 11.7244 | Train Loss Unweighted: 3.9925 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.1718 | Regression_cos_phi: 0.0487 | Regression_energy: 0.3212 | Regression_eta: 0.1125 | Regression_pt: 0.2729 | Regression_sin_phi: 0.0492 | Group Train Loss Unweighted (window mean): physical_pu: 4.0838 | physical_nopu: 5.2370 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3443 | Classification_binary: 3.1944 | Regression_cos_phi: 1.7872 | Regression_energy: 1.5670 | Regression_eta: 1.3436 | Regression_pt: 1.9514 | Regression_sin_phi: 1.7421 | LR: 3.99e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.2688s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0037 | max_forward: 0.0726 | max_loss: 0.0066 | max_backward: 0.1670 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4108.6 | uss_MB: 2925.8 | child_rss_MB: 10838.2 | allocated_MB: 406.4 | reserved_MB: 11904.0 | inactive_split_MB: 179.6 +rank7 [2026-10-27 16:25:55,983] INFO : Step 5000/100000 rank7 | Train Loss: 13.1313 | Train Loss Unweighted: 4.9481 | Train Loss Components: Classification: 0.0138 | Classification_binary: 4.0144 | Regression_cos_phi: 0.0518 | Regression_energy: 0.3902 | Regression_eta: 0.0928 | Regression_pt: 0.3380 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 4.0057 | physical_nopu: 5.1582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2952 | Classification_binary: 4.0686 | Regression_cos_phi: 1.4215 | Regression_energy: 1.8449 | Regression_eta: 1.4840 | Regression_pt: 2.4202 | Regression_sin_phi: 1.5484 | LR: 3.98e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.3570s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0196 | max_loss: 0.0037 | max_backward: 0.0324 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 1.3896 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0068 | Memory: rss_MB: 4091.1 | uss_MB: 2927.1 | child_rss_MB: 10673.2 | allocated_MB: 393.9 | reserved_MB: 11904.0 | inactive_split_MB: 90.0 +rank7 [2026-10-27 16:28:58,362] INFO : Step 6000/100000 rank7 | Train Loss: 14.5226 | Train Loss Unweighted: 5.0639 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.8601 | Regression_cos_phi: 0.0525 | Regression_energy: 0.5359 | Regression_eta: 0.0694 | Regression_pt: 0.4825 | Regression_sin_phi: 0.0459 | Group Train Loss Unweighted (window mean): physical_pu: 3.9666 | physical_nopu: 5.0829 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4383 | Classification_binary: 3.6282 | Regression_cos_phi: 1.5025 | Regression_energy: 2.6293 | Regression_eta: 0.6583 | Regression_pt: 3.6253 | Regression_sin_phi: 1.6072 | LR: 3.97e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.5989s | Phase Times: max_pre_step_barrier: 0.0280 | max_batch_to_device: 0.0008 | max_forward: 0.0311 | max_loss: 0.0037 | max_backward: 0.0629 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 2.6604 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0119 | Memory: rss_MB: 4078.1 | uss_MB: 2934.4 | child_rss_MB: 10495.3 | allocated_MB: 394.6 | reserved_MB: 12524.0 | inactive_split_MB: 109.0 +rank7 [2026-10-27 16:31:51,870] INFO : Step 7000/100000 rank7 | Train Loss: 13.9772 | Train Loss Unweighted: 5.0398 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.9636 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4671 | Regression_eta: 0.0797 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0537 | Group Train Loss Unweighted (window mean): physical_pu: 3.9390 | physical_nopu: 5.0316 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1903 | Classification_binary: 3.9236 | Regression_cos_phi: 1.2867 | Regression_energy: 3.6554 | Regression_eta: 0.8950 | Regression_pt: 5.2394 | Regression_sin_phi: 1.5409 | LR: 3.96e-04 | DataLoad Time: 0.0009s | Model Forward Time: 3.1759s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0243 | max_loss: 0.0036 | max_backward: 0.0295 | max_optimizer_step: 0.2429 | max_tensorboard_logging: 2.1406 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 4070.6 | uss_MB: 2937.8 | child_rss_MB: 10370.1 | allocated_MB: 393.9 | reserved_MB: 12524.0 | inactive_split_MB: 107.5 +rank7 [2026-10-27 16:35:03,086] INFO : Step 8000/100000 rank7 | Train Loss: 13.6161 | Train Loss Unweighted: 4.9650 | Train Loss Components: Classification: 0.0187 | Classification_binary: 3.9275 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4519 | Regression_eta: 0.0615 | Regression_pt: 0.4056 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.9031 | physical_nopu: 4.9878 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4127 | Classification_binary: 3.9681 | Regression_cos_phi: 2.2051 | Regression_energy: 2.3519 | Regression_eta: 0.7987 | Regression_pt: 3.1195 | Regression_sin_phi: 1.5941 | LR: 3.94e-04 | DataLoad Time: 0.0015s | Model Forward Time: 2.9575s | Phase Times: max_pre_step_barrier: 0.0692 | max_batch_to_device: 0.0893 | max_forward: 0.1169 | max_loss: 0.0037 | max_backward: 0.0488 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 1.9012 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4069.6 | uss_MB: 2945.4 | child_rss_MB: 10376.1 | allocated_MB: 393.8 | reserved_MB: 12524.0 | inactive_split_MB: 105.1 +rank7 [2026-10-27 16:38:07,477] INFO : Step 9000/100000 rank7 | Train Loss: 15.1606 | Train Loss Unweighted: 5.0855 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.7493 | Regression_cos_phi: 0.0474 | Regression_energy: 0.5909 | Regression_eta: 0.0749 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.8927 | physical_nopu: 4.9579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3879 | Classification_binary: 3.7716 | Regression_cos_phi: 1.6458 | Regression_energy: 2.7435 | Regression_eta: 0.7078 | Regression_pt: 3.7514 | Regression_sin_phi: 1.3642 | LR: 3.93e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.8505s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0007 | max_forward: 0.0386 | max_loss: 0.0041 | max_backward: 0.0582 | max_optimizer_step: 0.1496 | max_tensorboard_logging: 1.9225 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 4056.6 | uss_MB: 2948.9 | child_rss_MB: 10378.8 | allocated_MB: 395.8 | reserved_MB: 12524.0 | inactive_split_MB: 89.5 +rank7 [2026-10-27 16:41:18,311] INFO : Step 10000/100000 rank7 | Train Loss: 12.2525 | Train Loss Unweighted: 4.6725 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8667 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3374 | Regression_eta: 0.0750 | Regression_pt: 0.2772 | Regression_sin_phi: 0.0525 | Group Train Loss Unweighted (window mean): physical_pu: 3.8762 | physical_nopu: 4.9262 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2816 | Classification_binary: 3.9773 | Regression_cos_phi: 1.9270 | Regression_energy: 1.5873 | Regression_eta: 0.8671 | Regression_pt: 1.8882 | Regression_sin_phi: 1.9163 | LR: 3.91e-04 | DataLoad Time: 0.0013s | Model Forward Time: 3.1914s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0277 | max_loss: 0.0035 | max_backward: 0.0304 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 2.2107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4024.3 | uss_MB: 2953.8 | child_rss_MB: 10382.4 | allocated_MB: 393.6 | reserved_MB: 12524.0 | inactive_split_MB: 89.8 +rank7 [2026-10-27 16:41:19,252] INFO : Running validation on rank7 +rank7 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 16:41:19,252] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 16:43:24,652] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:43:25,081] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 16:43:52,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 16:43:53,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 16:43:53,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 16:43:54,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 16:43:54,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 16:43:55,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 16:43:55,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 16:43:55,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 16:44:05,970] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-27 16:44:06,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:44:06,938] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 16:44:22,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 16:44:22,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 16:44:23,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 16:44:23,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 16:44:23,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 16:44:24,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 16:44:24,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 16:44:25,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 16:44:35,357] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 16:44:35,783] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:44:36,273] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 16:44:51,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 16:44:51,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 16:44:52,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 16:44:52,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 16:44:53,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 16:44:53,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 16:44:54,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 16:44:54,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_10000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 16:45:04,482] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 16:48:51,455] INFO : Step 11000/100000 rank7 | Train Loss: 13.5331 | Train Loss Unweighted: 4.8490 | Train Loss Components: Classification: 0.0178 | Classification_binary: 3.8332 | Regression_cos_phi: 0.0514 | Regression_energy: 0.4326 | Regression_eta: 0.0715 | Regression_pt: 0.3906 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.8629 | physical_nopu: 4.8860 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3751 | Classification_binary: 3.8465 | Regression_cos_phi: 1.7611 | Regression_energy: 1.8298 | Regression_eta: 0.8820 | Regression_pt: 2.3955 | Regression_sin_phi: 1.8262 | LR: 3.89e-04 | DataLoad Time: 0.0027s | Model Forward Time: 3.1301s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0010 | max_forward: 0.0938 | max_loss: 0.0036 | max_backward: 0.1077 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.9405 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6748.2 | uss_MB: 5693.6 | child_rss_MB: 10378.2 | allocated_MB: 394.1 | reserved_MB: 11016.0 | inactive_split_MB: 102.9 +rank7 [2026-10-27 16:51:46,972] INFO : Step 12000/100000 rank7 | Train Loss: 16.0880 | Train Loss Unweighted: 5.2955 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.7102 | Regression_cos_phi: 0.0409 | Regression_energy: 0.7300 | Regression_eta: 0.0621 | Regression_pt: 0.6963 | Regression_sin_phi: 0.0392 | Group Train Loss Unweighted (window mean): physical_pu: 3.8522 | physical_nopu: 4.8676 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3729 | Classification_binary: 3.6863 | Regression_cos_phi: 1.3988 | Regression_energy: 3.1647 | Regression_eta: 0.9242 | Regression_pt: 4.4056 | Regression_sin_phi: 1.1731 | LR: 3.87e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.9527s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1050 | max_forward: 0.2073 | max_loss: 0.0038 | max_backward: 0.1040 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 1.6163 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6734.2 | uss_MB: 5695.7 | child_rss_MB: 10317.1 | allocated_MB: 394.0 | reserved_MB: 11384.0 | inactive_split_MB: 88.0 +rank7 [2026-10-27 16:54:47,130] INFO : Step 13000/100000 rank7 | Train Loss: 12.2526 | Train Loss Unweighted: 4.6994 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.8775 | Regression_cos_phi: 0.0480 | Regression_energy: 0.3426 | Regression_eta: 0.0688 | Regression_pt: 0.2970 | Regression_sin_phi: 0.0539 | Group Train Loss Unweighted (window mean): physical_pu: 3.8404 | physical_nopu: 4.8415 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2646 | Classification_binary: 3.8193 | Regression_cos_phi: 1.8706 | Regression_energy: 1.5056 | Regression_eta: 0.8037 | Regression_pt: 1.9258 | Regression_sin_phi: 1.8470 | LR: 3.85e-04 | DataLoad Time: 0.0057s | Model Forward Time: 3.3046s | Phase Times: max_pre_step_barrier: 0.0143 | max_batch_to_device: 0.0010 | max_forward: 0.0740 | max_loss: 0.0036 | max_backward: 0.0639 | max_optimizer_step: 0.2642 | max_tensorboard_logging: 2.1016 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 5345.9 | uss_MB: 4318.0 | child_rss_MB: 10308.5 | allocated_MB: 394.7 | reserved_MB: 12168.0 | inactive_split_MB: 111.5 +rank7 [2026-10-27 16:57:46,633] INFO : Step 14000/100000 rank7 | Train Loss: 11.1328 | Train Loss Unweighted: 3.8032 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.0595 | Regression_cos_phi: 0.0494 | Regression_energy: 0.2823 | Regression_eta: 0.1077 | Regression_pt: 0.2396 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.8281 | physical_nopu: 4.8178 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3509 | Classification_binary: 3.0654 | Regression_cos_phi: 1.7736 | Regression_energy: 1.3441 | Regression_eta: 1.1873 | Regression_pt: 1.6904 | Regression_sin_phi: 1.5573 | LR: 3.83e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.7659s | Phase Times: max_pre_step_barrier: 0.1064 | max_batch_to_device: 0.0033 | max_forward: 0.1130 | max_loss: 0.0036 | max_backward: 0.2788 | max_optimizer_step: 0.1663 | max_tensorboard_logging: 2.1758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1099 | Memory: rss_MB: 5335.9 | uss_MB: 4329.8 | child_rss_MB: 10259.4 | allocated_MB: 405.3 | reserved_MB: 12168.0 | inactive_split_MB: 126.3 +rank7 [2026-10-27 17:00:50,804] INFO : Step 15000/100000 rank7 | Train Loss: 12.4223 | Train Loss Unweighted: 4.6301 | Train Loss Components: Classification: 0.0132 | Classification_binary: 3.7975 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3484 | Regression_eta: 0.0765 | Regression_pt: 0.2883 | Regression_sin_phi: 0.0502 | Group Train Loss Unweighted (window mean): physical_pu: 3.8205 | physical_nopu: 4.8025 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3733 | Classification_binary: 3.8029 | Regression_cos_phi: 2.1154 | Regression_energy: 1.8011 | Regression_eta: 0.6278 | Regression_pt: 2.4708 | Regression_sin_phi: 1.8554 | LR: 3.80e-04 | DataLoad Time: 0.0007s | Model Forward Time: 3.2238s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0799 | max_loss: 0.0851 | max_backward: 0.1383 | max_optimizer_step: 0.1621 | max_tensorboard_logging: 2.1079 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5320.9 | uss_MB: 4324.4 | child_rss_MB: 10137.7 | allocated_MB: 394.1 | reserved_MB: 12168.0 | inactive_split_MB: 89.6 +rank7 [2026-10-27 17:04:12,219] INFO : Step 16000/100000 rank7 | Train Loss: 13.1559 | Train Loss Unweighted: 4.6950 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.6917 | Regression_cos_phi: 0.0473 | Regression_energy: 0.4330 | Regression_eta: 0.0771 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.8147 | physical_nopu: 4.7795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3407 | Classification_binary: 3.7840 | Regression_cos_phi: 2.1415 | Regression_energy: 2.0679 | Regression_eta: 0.9813 | Regression_pt: 2.4885 | Regression_sin_phi: 2.0330 | LR: 3.78e-04 | DataLoad Time: 0.0028s | Model Forward Time: 3.2749s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.0598 | max_loss: 0.0036 | max_backward: 0.1881 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 2.1012 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5304.0 | uss_MB: 4326.3 | child_rss_MB: 10027.9 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 88.9 +rank7 [2026-10-27 17:07:14,836] INFO : Step 17000/100000 rank7 | Train Loss: 13.6189 | Train Loss Unweighted: 4.8180 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7348 | Regression_cos_phi: 0.0492 | Regression_energy: 0.4756 | Regression_eta: 0.0731 | Regression_pt: 0.4225 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.8095 | physical_nopu: 4.7681 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2832 | Classification_binary: 3.7322 | Regression_cos_phi: 2.1786 | Regression_energy: 1.7183 | Regression_eta: 0.8562 | Regression_pt: 2.4048 | Regression_sin_phi: 1.6287 | LR: 3.75e-04 | DataLoad Time: 0.0016s | Model Forward Time: 4.0408s | Phase Times: max_pre_step_barrier: 0.6520 | max_batch_to_device: 0.0829 | max_forward: 0.1388 | max_loss: 0.0036 | max_backward: 0.0636 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 2.2240 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5273.0 | uss_MB: 4329.2 | child_rss_MB: 9776.7 | allocated_MB: 394.3 | reserved_MB: 12168.0 | inactive_split_MB: 88.4 +rank7 [2026-10-27 17:10:19,258] INFO : Step 18000/100000 rank7 | Train Loss: 12.6032 | Train Loss Unweighted: 4.6266 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.7426 | Regression_cos_phi: 0.0507 | Regression_energy: 0.3730 | Regression_eta: 0.0736 | Regression_pt: 0.3195 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7954 | physical_nopu: 4.7518 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3319 | Classification_binary: 3.7383 | Regression_cos_phi: 1.8878 | Regression_energy: 1.5521 | Regression_eta: 0.6797 | Regression_pt: 2.0589 | Regression_sin_phi: 2.4298 | LR: 3.72e-04 | DataLoad Time: 0.0019s | Model Forward Time: 4.2068s | Phase Times: max_pre_step_barrier: 0.0217 | max_batch_to_device: 0.0009 | max_forward: 0.1212 | max_loss: 0.0040 | max_backward: 0.1670 | max_optimizer_step: 0.1610 | max_tensorboard_logging: 3.0399 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5245.9 | uss_MB: 4331.6 | child_rss_MB: 9623.0 | allocated_MB: 394.0 | reserved_MB: 12168.0 | inactive_split_MB: 87.1 +rank7 [2026-10-27 17:13:21,531] INFO : Step 19000/100000 rank7 | Train Loss: 14.6123 | Train Loss Unweighted: 5.0036 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.6735 | Regression_cos_phi: 0.0398 | Regression_energy: 0.6019 | Regression_eta: 0.0612 | Regression_pt: 0.5669 | Regression_sin_phi: 0.0441 | Group Train Loss Unweighted (window mean): physical_pu: 3.7887 | physical_nopu: 4.7426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3210 | Classification_binary: 3.7120 | Regression_cos_phi: 1.4644 | Regression_energy: 2.9047 | Regression_eta: 0.8268 | Regression_pt: 3.9305 | Regression_sin_phi: 1.4327 | LR: 3.69e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.4962s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1593 | max_loss: 0.0094 | max_backward: 0.0749 | max_optimizer_step: 0.1658 | max_tensorboard_logging: 2.3024 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5218.9 | uss_MB: 4333.7 | child_rss_MB: 9466.4 | allocated_MB: 394.5 | reserved_MB: 12168.0 | inactive_split_MB: 103.1 +rank7 [2026-10-27 17:16:34,024] INFO : Step 20000/100000 rank7 | Train Loss: 13.1046 | Train Loss Unweighted: 4.7571 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.7799 | Regression_cos_phi: 0.0517 | Regression_energy: 0.4251 | Regression_eta: 0.0654 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0511 | Group Train Loss Unweighted (window mean): physical_pu: 3.7835 | physical_nopu: 4.7206 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2496 | Classification_binary: 3.9257 | Regression_cos_phi: 1.9103 | Regression_energy: 1.4055 | Regression_eta: 0.7139 | Regression_pt: 1.6985 | Regression_sin_phi: 1.7020 | LR: 3.66e-04 | DataLoad Time: 0.0032s | Model Forward Time: 3.2222s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0008 | max_forward: 0.1344 | max_loss: 0.0037 | max_backward: 0.0763 | max_optimizer_step: 0.1649 | max_tensorboard_logging: 1.9693 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5216.3 | uss_MB: 4336.2 | child_rss_MB: 9462.6 | allocated_MB: 392.8 | reserved_MB: 12168.0 | inactive_split_MB: 89.4 +rank7 [2026-10-27 17:16:34,969] INFO : Running validation on rank7 +rank7 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 17:16:34,969] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 17:18:27,486] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 17:18:27,879] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 17:18:53,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 17:18:54,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 17:18:54,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 17:18:55,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 17:18:55,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 17:18:56,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 17:18:56,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 17:18:57,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 17:19:07,120] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 17:19:07,312] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 17:19:07,735] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 17:19:22,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 17:19:23,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 17:19:23,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 17:19:24,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 17:19:24,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 17:19:25,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 17:19:25,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 17:19:26,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 17:19:36,110] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 17:19:36,428] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 17:19:36,813] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 17:19:51,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 17:19:52,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 17:19:52,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 17:19:52,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 17:19:53,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 17:19:53,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 17:19:54,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 17:19:54,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_20000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 17:20:04,774] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-27 17:23:25,419] INFO : Step 21000/100000 rank7 | Train Loss: 12.4320 | Train Loss Unweighted: 4.5235 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.6041 | Regression_cos_phi: 0.0444 | Regression_energy: 0.3902 | Regression_eta: 0.0812 | Regression_pt: 0.3383 | Regression_sin_phi: 0.0478 | Group Train Loss Unweighted (window mean): physical_pu: 3.7732 | physical_nopu: 4.7052 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3427 | Classification_binary: 3.5674 | Regression_cos_phi: 1.6982 | Regression_energy: 1.9589 | Regression_eta: 1.1431 | Regression_pt: 2.3406 | Regression_sin_phi: 1.9153 | LR: 3.62e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.7288s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.2105 | max_loss: 0.0056 | max_backward: 0.1347 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 2.3606 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5326.2 | uss_MB: 4454.3 | child_rss_MB: 9450.4 | allocated_MB: 394.6 | reserved_MB: 11062.0 | inactive_split_MB: 89.8 +rank7 [2026-10-27 17:26:14,701] INFO : Step 22000/100000 rank7 | Train Loss: 14.5232 | Train Loss Unweighted: 4.8893 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.6034 | Regression_cos_phi: 0.0476 | Regression_energy: 0.5714 | Regression_eta: 0.0738 | Regression_pt: 0.5351 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.7709 | physical_nopu: 4.6938 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3827 | Classification_binary: 3.6136 | Regression_cos_phi: 1.4065 | Regression_energy: 2.5903 | Regression_eta: 0.7483 | Regression_pt: 3.5740 | Regression_sin_phi: 1.2366 | LR: 3.59e-04 | DataLoad Time: 0.0028s | Model Forward Time: 2.6767s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0010 | max_forward: 0.0774 | max_loss: 0.0142 | max_backward: 0.0921 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.4442 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1027 | Memory: rss_MB: 5327.2 | uss_MB: 4458.5 | child_rss_MB: 9447.9 | allocated_MB: 394.2 | reserved_MB: 11488.0 | inactive_split_MB: 86.8 +rank7 [2026-10-27 17:29:50,014] INFO : Step 23000/100000 rank7 | Train Loss: 12.2140 | Train Loss Unweighted: 4.5907 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.7888 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3388 | Regression_eta: 0.0762 | Regression_pt: 0.2687 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.7690 | physical_nopu: 4.6810 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2667 | Classification_binary: 3.8970 | Regression_cos_phi: 2.1099 | Regression_energy: 1.5842 | Regression_eta: 1.0962 | Regression_pt: 1.9078 | Regression_sin_phi: 1.7427 | LR: 3.55e-04 | DataLoad Time: 0.0103s | Model Forward Time: 4.1184s | Phase Times: max_pre_step_barrier: 0.7590 | max_batch_to_device: 0.0007 | max_forward: 0.0763 | max_loss: 0.0189 | max_backward: 0.1117 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 2.0913 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5325.2 | uss_MB: 4459.8 | child_rss_MB: 9460.6 | allocated_MB: 394.0 | reserved_MB: 12220.0 | inactive_split_MB: 178.1 +rank7 [2026-10-27 17:33:22,189] INFO : Step 24000/100000 rank7 | Train Loss: 15.4786 | Train Loss Unweighted: 4.9908 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.5301 | Regression_cos_phi: 0.0467 | Regression_energy: 0.6682 | Regression_eta: 0.0624 | Regression_pt: 0.6223 | Regression_sin_phi: 0.0450 | Group Train Loss Unweighted (window mean): physical_pu: 3.7564 | physical_nopu: 4.6761 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3442 | Classification_binary: 3.7075 | Regression_cos_phi: 1.6606 | Regression_energy: 3.9414 | Regression_eta: 0.9184 | Regression_pt: 5.1709 | Regression_sin_phi: 1.5357 | LR: 3.51e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.9314s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0719 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1672 | max_tensorboard_logging: 1.8279 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5318.8 | uss_MB: 4462.5 | child_rss_MB: 9451.5 | allocated_MB: 393.8 | reserved_MB: 12220.0 | inactive_split_MB: 105.1 +rank7 [2026-10-27 17:36:26,064] INFO : Step 25000/100000 rank7 | Train Loss: 12.1246 | Train Loss Unweighted: 4.4783 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.7007 | Regression_cos_phi: 0.0551 | Regression_energy: 0.3196 | Regression_eta: 0.0694 | Regression_pt: 0.2642 | Regression_sin_phi: 0.0578 | Group Train Loss Unweighted (window mean): physical_pu: 3.7531 | physical_nopu: 4.6582 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2942 | Classification_binary: 3.5735 | Regression_cos_phi: 1.6852 | Regression_energy: 1.5323 | Regression_eta: 0.7285 | Regression_pt: 1.7931 | Regression_sin_phi: 1.8364 | LR: 3.47e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.0300s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.1917 | max_loss: 0.0039 | max_backward: 0.1029 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 5304.8 | uss_MB: 4464.1 | child_rss_MB: 9445.8 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 86.4 +rank7 [2026-10-27 17:39:30,100] INFO : Step 26000/100000 rank7 | Train Loss: 12.6920 | Train Loss Unweighted: 4.5479 | Train Loss Components: Classification: 0.0188 | Classification_binary: 3.6003 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4034 | Regression_eta: 0.0734 | Regression_pt: 0.3564 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.7511 | physical_nopu: 4.6505 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3743 | Classification_binary: 3.6349 | Regression_cos_phi: 1.7941 | Regression_energy: 1.7804 | Regression_eta: 0.8586 | Regression_pt: 2.4790 | Regression_sin_phi: 1.6953 | LR: 3.43e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.0942s | Phase Times: max_pre_step_barrier: 0.0479 | max_batch_to_device: 0.0008 | max_forward: 0.1278 | max_loss: 0.0037 | max_backward: 0.0877 | max_optimizer_step: 0.1868 | max_tensorboard_logging: 1.8908 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5292.2 | uss_MB: 4467.2 | child_rss_MB: 9407.7 | allocated_MB: 395.0 | reserved_MB: 12220.0 | inactive_split_MB: 115.0 +rank7 [2026-10-27 17:42:32,922] INFO : Step 27000/100000 rank7 | Train Loss: 16.2758 | Train Loss Unweighted: 5.1421 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5608 | Regression_cos_phi: 0.0448 | Regression_energy: 0.7180 | Regression_eta: 0.0780 | Regression_pt: 0.6791 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.7443 | physical_nopu: 4.6426 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3310 | Classification_binary: 3.5501 | Regression_cos_phi: 1.2082 | Regression_energy: 3.8945 | Regression_eta: 0.7877 | Regression_pt: 5.4558 | Regression_sin_phi: 1.5129 | LR: 3.39e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.0079s | Phase Times: max_pre_step_barrier: 0.0385 | max_batch_to_device: 0.0009 | max_forward: 0.0708 | max_loss: 0.0038 | max_backward: 0.0914 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.8878 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5283.2 | uss_MB: 4469.8 | child_rss_MB: 9400.6 | allocated_MB: 394.3 | reserved_MB: 12220.0 | inactive_split_MB: 87.9 +rank7 [2026-10-27 17:45:38,004] INFO : Step 28000/100000 rank7 | Train Loss: 11.9115 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6175 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3339 | Regression_eta: 0.0761 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0516 | Group Train Loss Unweighted (window mean): physical_pu: 3.7393 | physical_nopu: 4.6288 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2959 | Classification_binary: 3.8376 | Regression_cos_phi: 2.1496 | Regression_energy: 1.8132 | Regression_eta: 0.9750 | Regression_pt: 2.0994 | Regression_sin_phi: 2.0233 | LR: 3.35e-04 | DataLoad Time: 0.0019s | Model Forward Time: 4.8022s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0008 | max_forward: 0.0886 | max_loss: 0.0038 | max_backward: 0.1193 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 3.6285 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 5274.2 | uss_MB: 4146.2 | child_rss_MB: 9355.1 | allocated_MB: 394.0 | reserved_MB: 12414.0 | inactive_split_MB: 106.5 +rank7 [2026-10-27 17:48:54,715] INFO : Step 29000/100000 rank7 | Train Loss: 13.9221 | Train Loss Unweighted: 4.6615 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.5075 | Regression_cos_phi: 0.0491 | Regression_energy: 0.5008 | Regression_eta: 0.0786 | Regression_pt: 0.4589 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7405 | physical_nopu: 4.6233 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4131 | Classification_binary: 3.5512 | Regression_cos_phi: 1.7366 | Regression_energy: 2.7156 | Regression_eta: 0.9608 | Regression_pt: 3.6098 | Regression_sin_phi: 1.4597 | LR: 3.30e-04 | DataLoad Time: 0.0468s | Model Forward Time: 3.3362s | Phase Times: max_pre_step_barrier: 0.1677 | max_batch_to_device: 0.0009 | max_forward: 0.0754 | max_loss: 0.0038 | max_backward: 0.0825 | max_optimizer_step: 0.1673 | max_tensorboard_logging: 2.1272 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5252.2 | uss_MB: 4453.2 | child_rss_MB: 9260.7 | allocated_MB: 394.9 | reserved_MB: 12414.0 | inactive_split_MB: 107.1 +rank7 [2026-10-27 17:51:57,834] INFO : Step 30000/100000 rank7 | Train Loss: 12.4988 | Train Loss Unweighted: 4.5639 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.6431 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3981 | Regression_eta: 0.0722 | Regression_pt: 0.3405 | Regression_sin_phi: 0.0498 | Group Train Loss Unweighted (window mean): physical_pu: 3.7342 | physical_nopu: 4.6125 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3150 | Classification_binary: 3.6896 | Regression_cos_phi: 2.1058 | Regression_energy: 2.6845 | Regression_eta: 0.8727 | Regression_pt: 3.2200 | Regression_sin_phi: 2.2093 | LR: 3.26e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.4587s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0762 | max_loss: 0.0136 | max_backward: 0.0854 | max_optimizer_step: 0.1722 | max_tensorboard_logging: 1.3349 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5242.2 | uss_MB: 4475.0 | child_rss_MB: 9193.8 | allocated_MB: 393.7 | reserved_MB: 12480.0 | inactive_split_MB: 89.7 +rank7 [2026-10-27 17:51:58,780] INFO : Running validation on rank7 +rank7 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 17:51:58,780] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 17:51:58,781] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 17:51:58,781] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 17:53:53,313] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 17:53:53,721] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 17:54:18,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 17:54:18,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 17:54:19,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 17:54:19,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 17:54:20,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 17:54:20,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 17:54:21,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 17:54:21,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 17:54:31,824] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 17:54:31,840] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 17:54:32,213] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 17:54:47,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 17:54:47,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 17:54:48,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 17:54:48,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 17:54:49,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 17:54:49,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 17:54:50,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 17:54:50,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 17:55:00,629] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 17:55:00,637] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 17:55:01,038] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 17:55:15,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 17:55:16,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 17:55:16,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 17:55:17,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 17:55:17,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 17:55:18,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 17:55:18,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 17:55:19,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_30000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 17:55:29,146] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-27 17:58:47,285] INFO : Step 31000/100000 rank7 | Train Loss: 12.0027 | Train Loss Unweighted: 4.4503 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.5779 | Regression_cos_phi: 0.0460 | Regression_energy: 0.3731 | Regression_eta: 0.0704 | Regression_pt: 0.3230 | Regression_sin_phi: 0.0451 | Group Train Loss Unweighted (window mean): physical_pu: 3.7262 | physical_nopu: 4.6083 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3337 | Classification_binary: 3.6282 | Regression_cos_phi: 1.9074 | Regression_energy: 2.2229 | Regression_eta: 1.0386 | Regression_pt: 2.9390 | Regression_sin_phi: 1.8400 | LR: 3.21e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.5858s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.1458 | max_loss: 0.0037 | max_backward: 0.1256 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 2.3030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7018.1 | uss_MB: 6249.8 | child_rss_MB: 9197.9 | allocated_MB: 394.8 | reserved_MB: 11168.0 | inactive_split_MB: 96.2 +rank7 [2026-10-27 18:01:54,932] INFO : Step 32000/100000 rank7 | Train Loss: 13.5217 | Train Loss Unweighted: 4.6918 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.5357 | Regression_cos_phi: 0.0443 | Regression_energy: 0.5169 | Regression_eta: 0.0728 | Regression_pt: 0.4649 | Regression_sin_phi: 0.0409 | Group Train Loss Unweighted (window mean): physical_pu: 3.7264 | physical_nopu: 4.6006 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3807 | Classification_binary: 3.5850 | Regression_cos_phi: 1.5705 | Regression_energy: 2.3488 | Regression_eta: 0.6879 | Regression_pt: 3.2091 | Regression_sin_phi: 1.2775 | LR: 3.16e-04 | DataLoad Time: 0.0076s | Model Forward Time: 2.3908s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.0795 | max_loss: 0.0917 | max_backward: 0.1851 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.1890 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5310.0 | uss_MB: 4562.1 | child_rss_MB: 9157.4 | allocated_MB: 394.9 | reserved_MB: 11240.0 | inactive_split_MB: 92.3 +rank7 [2026-10-27 18:04:57,677] INFO : Step 33000/100000 rank7 | Train Loss: 11.0126 | Train Loss Unweighted: 3.7465 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9928 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2905 | Regression_eta: 0.1058 | Regression_pt: 0.2472 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.7276 | physical_nopu: 4.5909 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3031 | Classification_binary: 3.0213 | Regression_cos_phi: 1.7074 | Regression_energy: 1.3134 | Regression_eta: 1.0890 | Regression_pt: 1.6226 | Regression_sin_phi: 1.7008 | LR: 3.12e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.8906s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0027 | max_forward: 0.1127 | max_loss: 0.0150 | max_backward: 0.3396 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.4660 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5295.8 | uss_MB: 4291.2 | child_rss_MB: 9057.9 | allocated_MB: 404.8 | reserved_MB: 11922.0 | inactive_split_MB: 135.4 +rank7 [2026-10-27 18:07:50,121] INFO : Step 34000/100000 rank7 | Train Loss: 14.2400 | Train Loss Unweighted: 4.7634 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5051 | Regression_cos_phi: 0.0450 | Regression_energy: 0.5627 | Regression_eta: 0.0631 | Regression_pt: 0.5251 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7180 | physical_nopu: 4.5858 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2892 | Classification_binary: 3.5804 | Regression_cos_phi: 1.8570 | Regression_energy: 1.8405 | Regression_eta: 0.8356 | Regression_pt: 2.5401 | Regression_sin_phi: 1.5743 | LR: 3.07e-04 | DataLoad Time: 0.0012s | Model Forward Time: 2.7414s | Phase Times: max_pre_step_barrier: 0.1802 | max_batch_to_device: 0.0765 | max_forward: 0.1650 | max_loss: 0.0036 | max_backward: 0.0935 | max_optimizer_step: 0.2788 | max_tensorboard_logging: 1.2621 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5285.5 | uss_MB: 4563.5 | child_rss_MB: 9059.3 | allocated_MB: 395.5 | reserved_MB: 11922.0 | inactive_split_MB: 99.3 +rank7 [2026-10-27 18:10:53,637] INFO : Step 35000/100000 rank7 | Train Loss: 11.4005 | Train Loss Unweighted: 4.4142 | Train Loss Components: Classification: 0.0091 | Classification_binary: 3.6972 | Regression_cos_phi: 0.0530 | Regression_energy: 0.2984 | Regression_eta: 0.0671 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.7147 | physical_nopu: 4.5802 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1801 | Classification_binary: 3.5288 | Regression_cos_phi: 1.5119 | Regression_energy: 1.3638 | Regression_eta: 0.4999 | Regression_pt: 1.6506 | Regression_sin_phi: 1.3797 | LR: 3.02e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4160s | Phase Times: max_pre_step_barrier: 0.0682 | max_batch_to_device: 0.0008 | max_forward: 0.1243 | max_loss: 0.0038 | max_backward: 0.0883 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.1949 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5274.5 | uss_MB: 4566.4 | child_rss_MB: 9063.3 | allocated_MB: 393.8 | reserved_MB: 11922.0 | inactive_split_MB: 85.2 +rank7 [2026-10-27 18:13:50,577] INFO : Step 36000/100000 rank7 | Train Loss: 11.8438 | Train Loss Unweighted: 4.4232 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.5751 | Regression_cos_phi: 0.0431 | Regression_energy: 0.3555 | Regression_eta: 0.0641 | Regression_pt: 0.3210 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.7110 | physical_nopu: 4.5729 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4577 | Classification_binary: 3.4009 | Regression_cos_phi: 1.8742 | Regression_energy: 1.7866 | Regression_eta: 0.8444 | Regression_pt: 2.4012 | Regression_sin_phi: 1.7291 | LR: 2.97e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.4514s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.0993 | max_loss: 0.0066 | max_backward: 0.1760 | max_optimizer_step: 0.1735 | max_tensorboard_logging: 1.2166 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 5259.5 | uss_MB: 4568.3 | child_rss_MB: 9032.9 | allocated_MB: 395.0 | reserved_MB: 11922.0 | inactive_split_MB: 95.2 +rank7 [2026-10-27 18:16:45,596] INFO : Step 37000/100000 rank7 | Train Loss: 15.1390 | Train Loss Unweighted: 4.8166 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4396 | Regression_cos_phi: 0.0489 | Regression_energy: 0.6142 | Regression_eta: 0.0886 | Regression_pt: 0.5621 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7102 | physical_nopu: 4.5673 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3414 | Classification_binary: 3.4406 | Regression_cos_phi: 1.1804 | Regression_energy: 3.5407 | Regression_eta: 0.6268 | Regression_pt: 4.9291 | Regression_sin_phi: 1.0484 | LR: 2.91e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.2788s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1889 | max_loss: 0.0036 | max_backward: 0.0899 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 1.0901 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 5243.2 | uss_MB: 4569.9 | child_rss_MB: 9022.4 | allocated_MB: 394.3 | reserved_MB: 11922.0 | inactive_split_MB: 107.1 +rank7 [2026-10-27 18:19:42,075] INFO : Step 38000/100000 rank7 | Train Loss: 11.6256 | Train Loss Unweighted: 4.4424 | Train Loss Components: Classification: 0.0117 | Classification_binary: 3.6501 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3369 | Regression_eta: 0.0664 | Regression_pt: 0.2825 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.7131 | physical_nopu: 4.5682 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2317 | Classification_binary: 3.5737 | Regression_cos_phi: 1.5262 | Regression_energy: 1.5387 | Regression_eta: 0.7583 | Regression_pt: 1.8502 | Regression_sin_phi: 1.5870 | LR: 2.86e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.2502s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.0965 | max_loss: 0.0038 | max_backward: 0.1051 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5237.2 | uss_MB: 4571.0 | child_rss_MB: 9030.3 | allocated_MB: 393.3 | reserved_MB: 12082.0 | inactive_split_MB: 110.5 +rank7 [2026-10-27 18:22:37,812] INFO : Step 39000/100000 rank7 | Train Loss: 13.0208 | Train Loss Unweighted: 4.4928 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4758 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4360 | Regression_eta: 0.0793 | Regression_pt: 0.3879 | Regression_sin_phi: 0.0500 | Group Train Loss Unweighted (window mean): physical_pu: 3.7060 | physical_nopu: 4.5607 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3960 | Classification_binary: 3.4117 | Regression_cos_phi: 1.5938 | Regression_energy: 2.2473 | Regression_eta: 1.1248 | Regression_pt: 2.9761 | Regression_sin_phi: 1.7722 | LR: 2.81e-04 | DataLoad Time: 0.0015s | Model Forward Time: 2.2315s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.0864 | max_loss: 0.0038 | max_backward: 0.0877 | max_optimizer_step: 0.1700 | max_tensorboard_logging: 1.1418 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5224.7 | uss_MB: 4572.8 | child_rss_MB: 9008.2 | allocated_MB: 394.4 | reserved_MB: 12706.0 | inactive_split_MB: 84.9 +rank7 [2026-10-27 18:25:31,277] INFO : Step 40000/100000 rank7 | Train Loss: 14.7562 | Train Loss Unweighted: 4.9007 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5708 | Regression_cos_phi: 0.0459 | Regression_energy: 0.5954 | Regression_eta: 0.0747 | Regression_pt: 0.5553 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7006 | physical_nopu: 4.5579 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2375 | Classification_binary: 3.5751 | Regression_cos_phi: 1.7852 | Regression_energy: 3.5236 | Regression_eta: 0.8144 | Regression_pt: 4.8147 | Regression_sin_phi: 1.8191 | LR: 2.76e-04 | DataLoad Time: 0.0014s | Model Forward Time: 2.3078s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.0814 | max_loss: 0.0614 | max_backward: 0.1417 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 1.1378 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5222.7 | uss_MB: 4573.6 | child_rss_MB: 8992.8 | allocated_MB: 393.7 | reserved_MB: 12706.0 | inactive_split_MB: 107.7 +rank7 [2026-10-27 18:25:32,321] INFO : Running validation on rank7 +rank7 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 18:25:32,321] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 18:27:23,860] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 18:27:24,220] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 18:27:48,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 18:27:48,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 18:27:49,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 18:27:49,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 18:27:50,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 18:27:50,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 18:27:51,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 18:27:51,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 18:28:01,735] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 18:28:01,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 18:28:02,235] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 18:28:17,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 18:28:17,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 18:28:17,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 18:28:18,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 18:28:18,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 18:28:19,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 18:28:19,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 18:28:20,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 18:28:30,304] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 18:28:30,989] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 18:28:31,367] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 18:28:46,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 18:28:47,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 18:28:47,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 18:28:48,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 18:28:48,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 18:28:48,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 18:28:49,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 18:28:49,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_40000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 18:28:59,878] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 18:32:27,442] INFO : Step 41000/100000 rank7 | Train Loss: 12.3609 | Train Loss Unweighted: 4.4817 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5838 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3837 | Regression_eta: 0.0731 | Regression_pt: 0.3287 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.7016 | physical_nopu: 4.5433 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3323 | Classification_binary: 3.5622 | Regression_cos_phi: 1.6786 | Regression_energy: 1.5234 | Regression_eta: 0.8998 | Regression_pt: 1.8343 | Regression_sin_phi: 1.6770 | LR: 2.70e-04 | DataLoad Time: 0.0021s | Model Forward Time: 4.5169s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0009 | max_forward: 0.1338 | max_loss: 0.0038 | max_backward: 0.1857 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 3.1120 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 7028.8 | uss_MB: 6392.1 | child_rss_MB: 8998.3 | allocated_MB: 394.0 | reserved_MB: 11364.0 | inactive_split_MB: 106.1 +rank7 [2026-10-27 18:35:19,857] INFO : Step 42000/100000 rank7 | Train Loss: 12.9790 | Train Loss Unweighted: 4.5354 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4379 | Regression_cos_phi: 0.0458 | Regression_energy: 0.4856 | Regression_eta: 0.0625 | Regression_pt: 0.4500 | Regression_sin_phi: 0.0370 | Group Train Loss Unweighted (window mean): physical_pu: 3.6959 | physical_nopu: 4.5474 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3473 | Classification_binary: 3.5326 | Regression_cos_phi: 2.5456 | Regression_energy: 1.3740 | Regression_eta: 0.9634 | Regression_pt: 1.7241 | Regression_sin_phi: 1.6020 | LR: 2.65e-04 | DataLoad Time: 0.0115s | Model Forward Time: 2.6324s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0009 | max_forward: 0.1375 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 1.3492 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 7016.8 | uss_MB: 5332.4 | child_rss_MB: 8994.4 | allocated_MB: 394.2 | reserved_MB: 12150.0 | inactive_split_MB: 99.3 +rank7 [2026-10-27 18:38:35,925] INFO : Step 43000/100000 rank7 | Train Loss: 10.9006 | Train Loss Unweighted: 3.7057 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.9740 | Regression_cos_phi: 0.0473 | Regression_energy: 0.2786 | Regression_eta: 0.1094 | Regression_pt: 0.2322 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6942 | physical_nopu: 4.5421 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3860 | Classification_binary: 2.9418 | Regression_cos_phi: 1.7242 | Regression_energy: 1.4309 | Regression_eta: 1.2498 | Regression_pt: 1.8347 | Regression_sin_phi: 1.6911 | LR: 2.59e-04 | DataLoad Time: 0.0020s | Model Forward Time: 5.2291s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0039 | max_forward: 0.1100 | max_loss: 0.0637 | max_backward: 0.2303 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 3.9250 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5285.0 | uss_MB: 4686.1 | child_rss_MB: 8875.5 | allocated_MB: 406.8 | reserved_MB: 12150.0 | inactive_split_MB: 152.7 +rank7 [2026-10-27 18:41:39,775] INFO : Step 44000/100000 rank7 | Train Loss: 14.8629 | Train Loss Unweighted: 4.8399 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4856 | Regression_cos_phi: 0.0438 | Regression_energy: 0.6070 | Regression_eta: 0.0739 | Regression_pt: 0.5644 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6933 | physical_nopu: 4.5331 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3835 | Classification_binary: 3.5710 | Regression_cos_phi: 1.3712 | Regression_energy: 3.1968 | Regression_eta: 0.7061 | Regression_pt: 4.4135 | Regression_sin_phi: 1.2712 | LR: 2.54e-04 | DataLoad Time: 0.0120s | Model Forward Time: 2.6737s | Phase Times: max_pre_step_barrier: 0.0851 | max_batch_to_device: 0.0008 | max_forward: 0.0946 | max_loss: 0.0041 | max_backward: 0.0964 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.3558 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5284.0 | uss_MB: 4679.1 | child_rss_MB: 8885.5 | allocated_MB: 394.8 | reserved_MB: 12150.0 | inactive_split_MB: 94.9 +rank7 [2026-10-27 18:44:46,984] INFO : Step 45000/100000 rank7 | Train Loss: 12.9535 | Train Loss Unweighted: 4.5658 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.6115 | Regression_cos_phi: 0.0537 | Regression_energy: 0.4117 | Regression_eta: 0.0846 | Regression_pt: 0.3413 | Regression_sin_phi: 0.0531 | Group Train Loss Unweighted (window mean): physical_pu: 3.6892 | physical_nopu: 4.5283 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3034 | Classification_binary: 3.7762 | Regression_cos_phi: 2.0618 | Regression_energy: 3.6650 | Regression_eta: 0.8860 | Regression_pt: 4.8015 | Regression_sin_phi: 1.9651 | LR: 2.48e-04 | DataLoad Time: 0.0032s | Model Forward Time: 2.4256s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1077 | max_loss: 0.0040 | max_backward: 0.1457 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 1.2404 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 5275.4 | uss_MB: 4680.1 | child_rss_MB: 8893.2 | allocated_MB: 394.0 | reserved_MB: 13366.0 | inactive_split_MB: 74.2 +rank7 [2026-10-27 18:48:06,241] INFO : Step 46000/100000 rank7 | Train Loss: 12.3783 | Train Loss Unweighted: 4.4476 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.5252 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3923 | Regression_eta: 0.0679 | Regression_pt: 0.3506 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6869 | physical_nopu: 4.5281 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4073 | Classification_binary: 3.4750 | Regression_cos_phi: 1.5945 | Regression_energy: 1.9258 | Regression_eta: 0.6991 | Regression_pt: 2.5152 | Regression_sin_phi: 1.6421 | LR: 2.43e-04 | DataLoad Time: 0.0014s | Model Forward Time: 2.4376s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1021 | max_loss: 0.0036 | max_backward: 0.1895 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 1.1894 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 5269.4 | uss_MB: 4681.7 | child_rss_MB: 8897.5 | allocated_MB: 395.1 | reserved_MB: 13366.0 | inactive_split_MB: 103.1 +rank7 [2026-10-27 18:51:16,142] INFO : Step 47000/100000 rank7 | Train Loss: 14.5142 | Train Loss Unweighted: 4.7631 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.3922 | Regression_cos_phi: 0.0407 | Regression_energy: 0.6220 | Regression_eta: 0.0662 | Regression_pt: 0.5862 | Regression_sin_phi: 0.0406 | Group Train Loss Unweighted (window mean): physical_pu: 3.6853 | physical_nopu: 4.5234 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4280 | Classification_binary: 3.3209 | Regression_cos_phi: 1.3970 | Regression_energy: 3.4116 | Regression_eta: 0.5686 | Regression_pt: 4.8199 | Regression_sin_phi: 1.6840 | LR: 2.37e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.5377s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0667 | max_forward: 0.1426 | max_loss: 0.0037 | max_backward: 0.0873 | max_optimizer_step: 0.2591 | max_tensorboard_logging: 1.1650 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5267.4 | uss_MB: 4682.3 | child_rss_MB: 8896.3 | allocated_MB: 394.9 | reserved_MB: 14166.0 | inactive_split_MB: 95.2 +rank7 [2026-10-27 18:54:12,871] INFO : Step 48000/100000 rank7 | Train Loss: 11.2030 | Train Loss Unweighted: 4.3343 | Train Loss Components: Classification: 0.0101 | Classification_binary: 3.6447 | Regression_cos_phi: 0.0521 | Regression_energy: 0.2822 | Regression_eta: 0.0657 | Regression_pt: 0.2293 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6844 | physical_nopu: 4.5168 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2204 | Classification_binary: 3.5417 | Regression_cos_phi: 2.0604 | Regression_energy: 1.1971 | Regression_eta: 0.7041 | Regression_pt: 1.4535 | Regression_sin_phi: 1.8401 | LR: 2.31e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.2608s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.0058 | max_backward: 0.0852 | max_optimizer_step: 0.1712 | max_tensorboard_logging: 1.1613 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5253.4 | uss_MB: 4683.3 | child_rss_MB: 8890.9 | allocated_MB: 393.9 | reserved_MB: 14166.0 | inactive_split_MB: 101.2 +rank7 [2026-10-27 18:57:19,782] INFO : Step 49000/100000 rank7 | Train Loss: 12.4372 | Train Loss Unweighted: 4.4483 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.5320 | Regression_cos_phi: 0.0491 | Regression_energy: 0.3881 | Regression_eta: 0.0742 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6801 | physical_nopu: 4.5174 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4559 | Classification_binary: 3.4326 | Regression_cos_phi: 1.5502 | Regression_energy: 1.6460 | Regression_eta: 0.7865 | Regression_pt: 2.2620 | Regression_sin_phi: 1.5679 | LR: 2.26e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.4737s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0009 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1014 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 1.3340 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5240.2 | uss_MB: 4683.4 | child_rss_MB: 8815.5 | allocated_MB: 393.7 | reserved_MB: 14166.0 | inactive_split_MB: 100.8 +rank7 [2026-10-27 19:00:32,233] INFO : Step 50000/100000 rank7 | Train Loss: 13.3654 | Train Loss Unweighted: 4.5742 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4424 | Regression_cos_phi: 0.0426 | Regression_energy: 0.5013 | Regression_eta: 0.0683 | Regression_pt: 0.4573 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6814 | physical_nopu: 4.5132 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3476 | Classification_binary: 3.5631 | Regression_cos_phi: 1.4731 | Regression_energy: 2.8783 | Regression_eta: 0.9098 | Regression_pt: 3.5506 | Regression_sin_phi: 1.7919 | LR: 2.20e-04 | DataLoad Time: 0.0012s | Model Forward Time: 2.5867s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0010 | max_forward: 0.1892 | max_loss: 0.0115 | max_backward: 0.1333 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.2641 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0862 | Memory: rss_MB: 5234.2 | uss_MB: 4685.5 | child_rss_MB: 8779.3 | allocated_MB: 394.6 | reserved_MB: 14166.0 | inactive_split_MB: 91.2 +rank7 [2026-10-27 19:00:33,174] INFO : Running validation on rank7 +rank7 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 19:00:33,175] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 19:02:37,501] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 19:02:37,941] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 19:03:02,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 19:03:03,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 19:03:03,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 19:03:04,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 19:03:04,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 19:03:05,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 19:03:05,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 19:03:06,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 19:03:16,016] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 19:03:16,262] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 19:03:16,635] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 19:03:31,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 19:03:32,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 19:03:32,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 19:03:33,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 19:03:33,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 19:03:33,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 19:03:34,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 19:03:34,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 19:03:44,855] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 19:03:45,439] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 19:03:45,817] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 19:04:00,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 19:04:01,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 19:04:01,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 19:04:02,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 19:04:02,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 19:04:03,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 19:04:03,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 19:04:04,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_50000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 19:04:14,023] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 19:07:44,423] INFO : Step 51000/100000 rank7 | Train Loss: 12.0993 | Train Loss Unweighted: 4.3946 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5485 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3591 | Regression_eta: 0.0789 | Regression_pt: 0.2941 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6745 | physical_nopu: 4.5037 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2430 | Classification_binary: 3.5728 | Regression_cos_phi: 1.8577 | Regression_energy: 1.6248 | Regression_eta: 0.8208 | Regression_pt: 2.0412 | Regression_sin_phi: 1.7016 | LR: 2.14e-04 | DataLoad Time: 0.0102s | Model Forward Time: 3.3972s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0008 | max_forward: 0.1391 | max_loss: 0.0036 | max_backward: 0.1509 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 2.0683 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 7102.6 | uss_MB: 6555.6 | child_rss_MB: 8775.1 | allocated_MB: 393.6 | reserved_MB: 10952.0 | inactive_split_MB: 101.2 +rank7 [2026-10-27 19:10:55,345] INFO : Step 52000/100000 rank7 | Train Loss: 10.7195 | Train Loss Unweighted: 3.6688 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9553 | Regression_cos_phi: 0.0470 | Regression_energy: 0.2719 | Regression_eta: 0.1059 | Regression_pt: 0.2258 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6784 | physical_nopu: 4.5078 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3382 | Classification_binary: 2.9015 | Regression_cos_phi: 1.7039 | Regression_energy: 1.2532 | Regression_eta: 1.1191 | Regression_pt: 1.5542 | Regression_sin_phi: 1.5825 | LR: 2.09e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.5034s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0031 | max_forward: 0.1853 | max_loss: 0.0256 | max_backward: 0.2604 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 2.0239 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7092.6 | uss_MB: 6564.5 | child_rss_MB: 8773.5 | allocated_MB: 404.8 | reserved_MB: 11626.0 | inactive_split_MB: 256.4 +rank7 [2026-10-27 19:13:48,854] INFO : Step 53000/100000 rank7 | Train Loss: 11.7022 | Train Loss Unweighted: 4.3546 | Train Loss Components: Classification: 0.0123 | Classification_binary: 3.5594 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3321 | Regression_eta: 0.0730 | Regression_pt: 0.2795 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6748 | physical_nopu: 4.5005 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2206 | Classification_binary: 3.6202 | Regression_cos_phi: 1.4879 | Regression_energy: 2.0463 | Regression_eta: 0.6449 | Regression_pt: 2.5793 | Regression_sin_phi: 1.7973 | LR: 2.03e-04 | DataLoad Time: 0.0015s | Model Forward Time: 3.1506s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0954 | max_loss: 0.0037 | max_backward: 0.1065 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.9892 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 5255.7 | uss_MB: 4729.4 | child_rss_MB: 8773.6 | allocated_MB: 393.6 | reserved_MB: 11910.0 | inactive_split_MB: 100.6 +rank7 [2026-10-27 19:17:18,773] INFO : Step 54000/100000 rank7 | Train Loss: 12.5762 | Train Loss Unweighted: 4.3665 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0487 | Regression_energy: 0.4104 | Regression_eta: 0.0739 | Regression_pt: 0.3685 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6696 | physical_nopu: 4.5036 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3733 | Classification_binary: 3.4488 | Regression_cos_phi: 1.7954 | Regression_energy: 1.8069 | Regression_eta: 1.1794 | Regression_pt: 2.4607 | Regression_sin_phi: 1.7018 | LR: 1.97e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.3954s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.1022 | max_forward: 0.1953 | max_loss: 0.0037 | max_backward: 0.0975 | max_optimizer_step: 0.2772 | max_tensorboard_logging: 2.0527 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5248.7 | uss_MB: 4730.3 | child_rss_MB: 8744.8 | allocated_MB: 394.9 | reserved_MB: 11910.0 | inactive_split_MB: 118.2 +rank7 [2026-10-27 19:20:28,222] INFO : Step 55000/100000 rank7 | Train Loss: 13.4632 | Train Loss Unweighted: 4.6500 | Train Loss Components: Classification: 0.0134 | Classification_binary: 3.5428 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4921 | Regression_eta: 0.0679 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6720 | physical_nopu: 4.4919 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3057 | Classification_binary: 3.5825 | Regression_cos_phi: 1.5406 | Regression_energy: 1.7972 | Regression_eta: 0.7518 | Regression_pt: 2.2265 | Regression_sin_phi: 1.5403 | LR: 1.92e-04 | DataLoad Time: 0.1107s | Model Forward Time: 3.6933s | Phase Times: max_pre_step_barrier: 0.1980 | max_batch_to_device: 0.0007 | max_forward: 0.0889 | max_loss: 0.0037 | max_backward: 0.2091 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 2.3779 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5235.8 | uss_MB: 4729.6 | child_rss_MB: 8640.7 | allocated_MB: 393.8 | reserved_MB: 11910.0 | inactive_split_MB: 89.1 +rank7 [2026-10-27 19:23:30,605] INFO : Step 56000/100000 rank7 | Train Loss: 11.6078 | Train Loss Unweighted: 4.2962 | Train Loss Components: Classification: 0.0140 | Classification_binary: 3.4759 | Regression_cos_phi: 0.0438 | Regression_energy: 0.3484 | Regression_eta: 0.0671 | Regression_pt: 0.2977 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6692 | physical_nopu: 4.4908 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4208 | Classification_binary: 3.4038 | Regression_cos_phi: 1.7601 | Regression_energy: 1.8580 | Regression_eta: 0.6970 | Regression_pt: 2.4325 | Regression_sin_phi: 1.5223 | LR: 1.86e-04 | DataLoad Time: 0.0308s | Model Forward Time: 3.2006s | Phase Times: max_pre_step_barrier: 0.0941 | max_batch_to_device: 0.0008 | max_forward: 0.0930 | max_loss: 0.0038 | max_backward: 0.0988 | max_optimizer_step: 0.1756 | max_tensorboard_logging: 1.8920 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 5221.8 | uss_MB: 4732.9 | child_rss_MB: 8582.9 | allocated_MB: 393.5 | reserved_MB: 11910.0 | inactive_split_MB: 115.7 +rank7 [2026-10-27 19:26:52,902] INFO : Step 57000/100000 rank7 | Train Loss: 13.9252 | Train Loss Unweighted: 4.6468 | Train Loss Components: Classification: 0.0167 | Classification_binary: 3.4640 | Regression_cos_phi: 0.0464 | Regression_energy: 0.5144 | Regression_eta: 0.0781 | Regression_pt: 0.4791 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6687 | physical_nopu: 4.4875 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2836 | Classification_binary: 3.5822 | Regression_cos_phi: 1.5978 | Regression_energy: 2.2979 | Regression_eta: 0.7517 | Regression_pt: 3.0684 | Regression_sin_phi: 1.6294 | LR: 1.81e-04 | DataLoad Time: 0.0025s | Model Forward Time: 4.6258s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.1223 | max_forward: 0.2094 | max_loss: 0.0420 | max_backward: 0.1628 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 2.9072 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 5216.8 | uss_MB: 4734.2 | child_rss_MB: 8511.2 | allocated_MB: 394.3 | reserved_MB: 12240.0 | inactive_split_MB: 101.4 +rank7 [2026-10-27 19:30:07,651] INFO : Step 58000/100000 rank7 | Train Loss: 11.5910 | Train Loss Unweighted: 4.3047 | Train Loss Components: Classification: 0.0106 | Classification_binary: 3.5490 | Regression_cos_phi: 0.0544 | Regression_energy: 0.3108 | Regression_eta: 0.0631 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0510 | Group Train Loss Unweighted (window mean): physical_pu: 3.6630 | physical_nopu: 4.4842 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.1948 | Classification_binary: 3.5019 | Regression_cos_phi: 2.0703 | Regression_energy: 1.1776 | Regression_eta: 0.5756 | Regression_pt: 1.4390 | Regression_sin_phi: 1.8561 | LR: 1.75e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.5121s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0007 | max_forward: 0.0898 | max_loss: 0.1080 | max_backward: 0.1964 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.2749 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 5209.3 | uss_MB: 4734.4 | child_rss_MB: 8432.6 | allocated_MB: 393.9 | reserved_MB: 12240.0 | inactive_split_MB: 121.9 +rank7 [2026-10-27 19:33:12,149] INFO : Step 59000/100000 rank7 | Train Loss: 12.2218 | Train Loss Unweighted: 4.4115 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.5166 | Regression_cos_phi: 0.0458 | Regression_energy: 0.3794 | Regression_eta: 0.0771 | Regression_pt: 0.3280 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6650 | physical_nopu: 4.4788 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3906 | Classification_binary: 3.6195 | Regression_cos_phi: 1.8263 | Regression_energy: 2.3275 | Regression_eta: 0.9375 | Regression_pt: 2.9086 | Regression_sin_phi: 2.1269 | LR: 1.70e-04 | DataLoad Time: 0.5928s | Model Forward Time: 2.6299s | Phase Times: max_pre_step_barrier: 0.6889 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0499 | max_backward: 0.1549 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5201.3 | uss_MB: 4735.4 | child_rss_MB: 8387.3 | allocated_MB: 394.0 | reserved_MB: 12240.0 | inactive_split_MB: 135.3 +rank7 [2026-10-27 19:36:26,517] INFO : Step 60000/100000 rank7 | Train Loss: 14.3810 | Train Loss Unweighted: 4.7229 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.4146 | Regression_cos_phi: 0.0449 | Regression_energy: 0.5841 | Regression_eta: 0.0716 | Regression_pt: 0.5495 | Regression_sin_phi: 0.0427 | Group Train Loss Unweighted (window mean): physical_pu: 3.6604 | physical_nopu: 4.4865 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4595 | Classification_binary: 3.2963 | Regression_cos_phi: 1.0814 | Regression_energy: 3.0481 | Regression_eta: 0.5753 | Regression_pt: 4.2545 | Regression_sin_phi: 1.0833 | LR: 1.64e-04 | DataLoad Time: 0.0032s | Model Forward Time: 2.4558s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.0849 | max_loss: 0.0036 | max_backward: 0.1478 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 1.2880 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5191.9 | uss_MB: 4495.0 | child_rss_MB: 8328.9 | allocated_MB: 394.8 | reserved_MB: 12240.0 | inactive_split_MB: 87.0 +rank7 [2026-10-27 19:36:27,558] INFO : Running validation on rank7 +rank7 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 19:36:27,559] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 19:38:18,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 19:38:19,028] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 19:38:43,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 19:38:44,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 19:38:44,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 19:38:45,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 19:38:45,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 19:38:46,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 19:38:46,521] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 19:38:47,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 19:38:57,015] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 19:38:57,241] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 19:38:57,609] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 19:39:12,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 19:39:12,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 19:39:13,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 19:39:13,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 19:39:14,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 19:39:14,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 19:39:15,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 19:39:15,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 19:39:25,528] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 19:39:26,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 19:39:26,533] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 19:39:41,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 19:39:42,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 19:39:42,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 19:39:42,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 19:39:43,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 19:39:43,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 19:39:44,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 19:39:44,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_60000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 19:39:54,712] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-27 19:43:33,543] INFO : Step 61000/100000 rank7 | Train Loss: 11.4262 | Train Loss Unweighted: 4.3706 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.6500 | Regression_cos_phi: 0.0531 | Regression_energy: 0.2956 | Regression_eta: 0.0675 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6553 | physical_nopu: 4.4780 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2990 | Classification_binary: 3.5418 | Regression_cos_phi: 1.7764 | Regression_energy: 1.2566 | Regression_eta: 0.6687 | Regression_pt: 1.5574 | Regression_sin_phi: 1.4482 | LR: 1.59e-04 | DataLoad Time: 0.0085s | Model Forward Time: 3.1978s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1116 | max_loss: 0.0036 | max_backward: 0.1506 | max_optimizer_step: 0.1828 | max_tensorboard_logging: 1.9738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 5784.0 | uss_MB: 5329.2 | child_rss_MB: 8302.4 | allocated_MB: 393.5 | reserved_MB: 11190.0 | inactive_split_MB: 103.1 +rank7 [2026-10-27 19:46:49,589] INFO : Step 62000/100000 rank7 | Train Loss: 10.7781 | Train Loss Unweighted: 3.6510 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9253 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2791 | Regression_eta: 0.1043 | Regression_pt: 0.2316 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4795 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3383 | Classification_binary: 2.9667 | Regression_cos_phi: 1.7682 | Regression_energy: 1.3285 | Regression_eta: 1.1142 | Regression_pt: 1.6077 | Regression_sin_phi: 1.6138 | LR: 1.54e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.6555s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0030 | max_forward: 0.1294 | max_loss: 0.0038 | max_backward: 0.2007 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.2115 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0496 | Memory: rss_MB: 5775.0 | uss_MB: 5052.2 | child_rss_MB: 8186.6 | allocated_MB: 404.4 | reserved_MB: 11190.0 | inactive_split_MB: 148.6 +rank7 [2026-10-27 19:50:02,443] INFO : Step 63000/100000 rank7 | Train Loss: 11.5766 | Train Loss Unweighted: 4.3655 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.6000 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3204 | Regression_eta: 0.0727 | Regression_pt: 0.2610 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6589 | physical_nopu: 4.4771 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2979 | Classification_binary: 3.6084 | Regression_cos_phi: 1.5911 | Regression_energy: 1.1185 | Regression_eta: 0.6142 | Regression_pt: 1.3836 | Regression_sin_phi: 1.3422 | LR: 1.49e-04 | DataLoad Time: 0.0012s | Model Forward Time: 2.3901s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0007 | max_forward: 0.1036 | max_loss: 0.0037 | max_backward: 0.1140 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2208 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 5773.0 | uss_MB: 5331.6 | child_rss_MB: 8138.8 | allocated_MB: 393.8 | reserved_MB: 11190.0 | inactive_split_MB: 93.0 +rank7 [2026-10-27 19:53:17,771] INFO : Step 64000/100000 rank7 | Train Loss: 13.2709 | Train Loss Unweighted: 4.5542 | Train Loss Components: Classification: 0.0175 | Classification_binary: 3.4295 | Regression_cos_phi: 0.0429 | Regression_energy: 0.4948 | Regression_eta: 0.0672 | Regression_pt: 0.4587 | Regression_sin_phi: 0.0436 | Group Train Loss Unweighted (window mean): physical_pu: 3.6551 | physical_nopu: 4.4714 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3719 | Classification_binary: 3.3469 | Regression_cos_phi: 1.5424 | Regression_energy: 3.1345 | Regression_eta: 0.7908 | Regression_pt: 4.4438 | Regression_sin_phi: 1.4437 | LR: 1.43e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.3960s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0928 | max_loss: 0.0570 | max_backward: 0.1589 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 1.2035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 5289.6 | uss_MB: 4857.6 | child_rss_MB: 8120.4 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank7 [2026-10-27 19:56:23,053] INFO : Step 65000/100000 rank7 | Train Loss: 15.6652 | Train Loss Unweighted: 4.9833 | Train Loss Components: Classification: 0.0144 | Classification_binary: 3.5112 | Regression_cos_phi: 0.0496 | Regression_energy: 0.6688 | Regression_eta: 0.0629 | Regression_pt: 0.6290 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6554 | physical_nopu: 4.4650 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3144 | Classification_binary: 3.2347 | Regression_cos_phi: 1.3870 | Regression_energy: 2.2027 | Regression_eta: 0.5425 | Regression_pt: 2.9296 | Regression_sin_phi: 1.1611 | LR: 1.38e-04 | DataLoad Time: 0.0015s | Model Forward Time: 2.7633s | Phase Times: max_pre_step_barrier: 0.1158 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0042 | max_backward: 0.1974 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.4176 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5282.6 | uss_MB: 4858.3 | child_rss_MB: 8025.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 124.9 +rank7 [2026-10-27 19:59:28,722] INFO : Step 66000/100000 rank7 | Train Loss: 12.0073 | Train Loss Unweighted: 4.3350 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4949 | Regression_cos_phi: 0.0509 | Regression_energy: 0.3535 | Regression_eta: 0.0691 | Regression_pt: 0.3018 | Regression_sin_phi: 0.0501 | Group Train Loss Unweighted (window mean): physical_pu: 3.6526 | physical_nopu: 4.4657 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2824 | Classification_binary: 3.5163 | Regression_cos_phi: 2.0711 | Regression_energy: 1.6105 | Regression_eta: 0.8883 | Regression_pt: 1.9619 | Regression_sin_phi: 1.9854 | LR: 1.33e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.4468s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0891 | max_loss: 0.0538 | max_backward: 0.1468 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 1.2753 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5277.7 | uss_MB: 4858.9 | child_rss_MB: 7982.1 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 107.1 +rank7 [2026-10-27 20:02:51,910] INFO : Step 67000/100000 rank7 | Train Loss: 13.3177 | Train Loss Unweighted: 4.5554 | Train Loss Components: Classification: 0.0166 | Classification_binary: 3.4800 | Regression_cos_phi: 0.0500 | Regression_energy: 0.4690 | Regression_eta: 0.0673 | Regression_pt: 0.4253 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6538 | physical_nopu: 4.4665 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3424 | Classification_binary: 3.4643 | Regression_cos_phi: 1.7157 | Regression_energy: 2.7560 | Regression_eta: 0.8237 | Regression_pt: 3.6730 | Regression_sin_phi: 1.4307 | LR: 1.28e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.9379s | Phase Times: max_pre_step_barrier: 0.0792 | max_batch_to_device: 0.0009 | max_forward: 0.0886 | max_loss: 0.1057 | max_backward: 0.1987 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 2.6109 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 5271.7 | uss_MB: 4859.8 | child_rss_MB: 7952.8 | allocated_MB: 393.9 | reserved_MB: 11792.0 | inactive_split_MB: 111.5 +rank7 [2026-10-27 20:06:09,103] INFO : Step 68000/100000 rank7 | Train Loss: 11.1535 | Train Loss Unweighted: 4.2643 | Train Loss Components: Classification: 0.0109 | Classification_binary: 3.5138 | Regression_cos_phi: 0.0456 | Regression_energy: 0.3163 | Regression_eta: 0.0648 | Regression_pt: 0.2659 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6527 | physical_nopu: 4.4693 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2870 | Classification_binary: 3.5053 | Regression_cos_phi: 1.4380 | Regression_energy: 1.7934 | Regression_eta: 0.4812 | Regression_pt: 2.3665 | Regression_sin_phi: 1.4835 | LR: 1.24e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.3894s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1008 | max_loss: 0.0767 | max_backward: 0.1768 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.1759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5263.4 | uss_MB: 4860.3 | child_rss_MB: 7883.7 | allocated_MB: 394.5 | reserved_MB: 11792.0 | inactive_split_MB: 88.6 +rank7 [2026-10-27 20:09:12,492] INFO : Step 69000/100000 rank7 | Train Loss: 12.1728 | Train Loss Unweighted: 4.3418 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4719 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3660 | Regression_eta: 0.0763 | Regression_pt: 0.3115 | Regression_sin_phi: 0.0548 | Group Train Loss Unweighted (window mean): physical_pu: 3.6494 | physical_nopu: 4.4629 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2730 | Classification_binary: 3.3904 | Regression_cos_phi: 1.5776 | Regression_energy: 1.7633 | Regression_eta: 0.8705 | Regression_pt: 2.2933 | Regression_sin_phi: 2.0645 | LR: 1.19e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.2954s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.1094 | max_loss: 0.0147 | max_backward: 0.1167 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 1.1204 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5251.4 | uss_MB: 4861.5 | child_rss_MB: 7816.8 | allocated_MB: 393.8 | reserved_MB: 11792.0 | inactive_split_MB: 106.5 +rank7 [2026-10-27 20:12:13,498] INFO : Step 70000/100000 rank7 | Train Loss: 14.9156 | Train Loss Unweighted: 4.7730 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.3975 | Regression_cos_phi: 0.0487 | Regression_energy: 0.6183 | Regression_eta: 0.0638 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0448 | Group Train Loss Unweighted (window mean): physical_pu: 3.6488 | physical_nopu: 4.4584 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3549 | Classification_binary: 3.4389 | Regression_cos_phi: 1.9645 | Regression_energy: 2.5723 | Regression_eta: 0.7237 | Regression_pt: 3.3651 | Regression_sin_phi: 1.7509 | LR: 1.14e-04 | DataLoad Time: 0.0069s | Model Forward Time: 2.3867s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1038 | max_loss: 0.0858 | max_backward: 0.1871 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 1.1619 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 5245.8 | uss_MB: 4860.8 | child_rss_MB: 7775.8 | allocated_MB: 395.0 | reserved_MB: 11792.0 | inactive_split_MB: 111.9 +rank7 [2026-10-27 20:12:14,505] INFO : Running validation on rank7 +rank7 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 20:12:14,505] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 20:14:09,433] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 20:14:09,864] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 20:14:34,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 20:14:35,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 20:14:35,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 20:14:36,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 20:14:36,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 20:14:36,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 20:14:37,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 20:14:37,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 20:14:47,935] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 20:14:47,942] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 20:14:48,316] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 20:15:03,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 20:15:03,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 20:15:04,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 20:15:04,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 20:15:05,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 20:15:05,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 20:15:06,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 20:15:06,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 20:15:16,786] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 20:15:17,039] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 20:15:17,440] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 20:15:32,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 20:15:32,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 20:15:33,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 20:15:33,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 20:15:34,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 20:15:34,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 20:15:35,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 20:15:35,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_70000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 20:15:45,520] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 20:19:21,751] INFO : Step 71000/100000 rank7 | Train Loss: 10.9922 | Train Loss Unweighted: 3.6772 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9201 | Regression_cos_phi: 0.0477 | Regression_energy: 0.2906 | Regression_eta: 0.1066 | Regression_pt: 0.2485 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6484 | physical_nopu: 4.4561 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3517 | Classification_binary: 2.8943 | Regression_cos_phi: 1.7199 | Regression_energy: 1.4774 | Regression_eta: 1.1609 | Regression_pt: 1.9382 | Regression_sin_phi: 1.6573 | LR: 1.10e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.5361s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0652 | max_forward: 0.2596 | max_loss: 0.0036 | max_backward: 0.3497 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 1.8772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 7279.8 | uss_MB: 6900.2 | child_rss_MB: 7804.4 | allocated_MB: 405.9 | reserved_MB: 11130.0 | inactive_split_MB: 135.5 +rank7 [2026-10-27 20:22:34,777] INFO : Step 72000/100000 rank7 | Train Loss: 14.4733 | Train Loss Unweighted: 4.6898 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3667 | Regression_cos_phi: 0.0458 | Regression_energy: 0.5948 | Regression_eta: 0.0650 | Regression_pt: 0.5567 | Regression_sin_phi: 0.0444 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4536 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3409 | Classification_binary: 3.3920 | Regression_cos_phi: 1.3399 | Regression_energy: 2.2595 | Regression_eta: 0.5711 | Regression_pt: 3.1169 | Regression_sin_phi: 1.1526 | LR: 1.05e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.5072s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1390 | max_loss: 0.0037 | max_backward: 0.1007 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 1.2476 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1281 | Memory: rss_MB: 5316.2 | uss_MB: 4936.1 | child_rss_MB: 7727.0 | allocated_MB: 393.8 | reserved_MB: 11706.0 | inactive_split_MB: 88.5 +rank7 [2026-10-27 20:25:34,428] INFO : Step 73000/100000 rank7 | Train Loss: 11.7903 | Train Loss Unweighted: 4.2561 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.5010 | Regression_cos_phi: 0.0578 | Regression_energy: 0.3092 | Regression_eta: 0.0644 | Regression_pt: 0.2565 | Regression_sin_phi: 0.0557 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4567 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2644 | Classification_binary: 3.7683 | Regression_cos_phi: 2.6488 | Regression_energy: 1.4252 | Regression_eta: 0.6570 | Regression_pt: 1.7419 | Regression_sin_phi: 1.9908 | LR: 1.01e-04 | DataLoad Time: 0.0012s | Model Forward Time: 2.6074s | Phase Times: max_pre_step_barrier: 0.1246 | max_batch_to_device: 0.0008 | max_forward: 0.0998 | max_loss: 0.0122 | max_backward: 0.1972 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.2348 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5307.2 | uss_MB: 4937.3 | child_rss_MB: 7690.0 | allocated_MB: 393.5 | reserved_MB: 11706.0 | inactive_split_MB: 93.2 +rank7 [2026-10-27 20:28:47,370] INFO : Step 74000/100000 rank7 | Train Loss: 11.6748 | Train Loss Unweighted: 4.2803 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4309 | Regression_cos_phi: 0.0428 | Regression_energy: 0.3572 | Regression_eta: 0.0695 | Regression_pt: 0.3177 | Regression_sin_phi: 0.0458 | Group Train Loss Unweighted (window mean): physical_pu: 3.6485 | physical_nopu: 4.4502 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4158 | Classification_binary: 3.5676 | Regression_cos_phi: 1.7623 | Regression_energy: 1.8721 | Regression_eta: 1.0976 | Regression_pt: 2.4661 | Regression_sin_phi: 1.9114 | LR: 9.68e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.0485s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1195 | max_loss: 0.0060 | max_backward: 0.2082 | max_optimizer_step: 0.1763 | max_tensorboard_logging: 1.8023 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5300.9 | uss_MB: 4938.3 | child_rss_MB: 7614.5 | allocated_MB: 394.3 | reserved_MB: 11706.0 | inactive_split_MB: 97.0 +rank7 [2026-10-27 20:31:56,443] INFO : Step 75000/100000 rank7 | Train Loss: 14.1604 | Train Loss Unweighted: 4.6445 | Train Loss Components: Classification: 0.0151 | Classification_binary: 3.4007 | Regression_cos_phi: 0.0489 | Regression_energy: 0.5577 | Regression_eta: 0.0640 | Regression_pt: 0.5112 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.6489 | physical_nopu: 4.4484 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3974 | Classification_binary: 3.3795 | Regression_cos_phi: 2.0310 | Regression_energy: 2.6915 | Regression_eta: 0.9205 | Regression_pt: 3.6421 | Regression_sin_phi: 1.6979 | LR: 9.27e-05 | DataLoad Time: 0.0031s | Model Forward Time: 2.9320s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0831 | max_loss: 0.0038 | max_backward: 0.2058 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.5751 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1279 | Memory: rss_MB: 5290.9 | uss_MB: 4938.0 | child_rss_MB: 7581.3 | allocated_MB: 394.4 | reserved_MB: 11706.0 | inactive_split_MB: 89.1 +rank7 [2026-10-27 20:34:57,734] INFO : Step 76000/100000 rank7 | Train Loss: 11.4149 | Train Loss Unweighted: 4.2984 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5224 | Regression_cos_phi: 0.0478 | Regression_energy: 0.3298 | Regression_eta: 0.0709 | Regression_pt: 0.2685 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6454 | physical_nopu: 4.4486 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2593 | Classification_binary: 3.3863 | Regression_cos_phi: 1.4657 | Regression_energy: 1.1549 | Regression_eta: 0.4417 | Regression_pt: 1.4866 | Regression_sin_phi: 1.6379 | LR: 8.88e-05 | DataLoad Time: 0.0011s | Model Forward Time: 3.1739s | Phase Times: max_pre_step_barrier: 0.0296 | max_batch_to_device: 0.0996 | max_forward: 0.1816 | max_loss: 0.0036 | max_backward: 0.0930 | max_optimizer_step: 0.2829 | max_tensorboard_logging: 1.7104 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1287 | Memory: rss_MB: 5286.0 | uss_MB: 4937.3 | child_rss_MB: 7586.8 | allocated_MB: 393.6 | reserved_MB: 12330.0 | inactive_split_MB: 97.7 +rank7 [2026-10-27 20:38:16,884] INFO : Step 77000/100000 rank7 | Train Loss: 12.8789 | Train Loss Unweighted: 4.4653 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4333 | Regression_cos_phi: 0.0478 | Regression_energy: 0.4517 | Regression_eta: 0.0624 | Regression_pt: 0.4079 | Regression_sin_phi: 0.0461 | Group Train Loss Unweighted (window mean): physical_pu: 3.6433 | physical_nopu: 4.4448 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2626 | Classification_binary: 3.3877 | Regression_cos_phi: 1.9864 | Regression_energy: 1.6419 | Regression_eta: 0.7197 | Regression_pt: 2.0231 | Regression_sin_phi: 1.7854 | LR: 8.50e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.0358s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.0831 | max_loss: 0.0514 | max_backward: 0.1524 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.8652 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5273.0 | uss_MB: 4939.8 | child_rss_MB: 7463.9 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 99.2 +rank7 [2026-10-27 20:41:21,661] INFO : Step 78000/100000 rank7 | Train Loss: 13.2420 | Train Loss Unweighted: 4.6345 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.5815 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4598 | Regression_eta: 0.0679 | Regression_pt: 0.4144 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6415 | physical_nopu: 4.4494 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2263 | Classification_binary: 3.6853 | Regression_cos_phi: 1.5314 | Regression_energy: 1.5640 | Regression_eta: 0.6525 | Regression_pt: 1.9459 | Regression_sin_phi: 1.3795 | LR: 8.13e-05 | DataLoad Time: 0.0011s | Model Forward Time: 3.0727s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0927 | max_loss: 0.0036 | max_backward: 0.2060 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.8450 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 5261.0 | uss_MB: 4940.8 | child_rss_MB: 7383.4 | allocated_MB: 394.7 | reserved_MB: 12330.0 | inactive_split_MB: 95.1 +rank7 [2026-10-27 20:44:31,405] INFO : Step 79000/100000 rank7 | Train Loss: 11.5047 | Train Loss Unweighted: 4.2761 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4919 | Regression_cos_phi: 0.0474 | Regression_energy: 0.3261 | Regression_eta: 0.0640 | Regression_pt: 0.2834 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6416 | physical_nopu: 4.4460 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3531 | Classification_binary: 3.2793 | Regression_cos_phi: 1.6724 | Regression_energy: 1.6936 | Regression_eta: 0.4811 | Regression_pt: 2.2358 | Regression_sin_phi: 1.5752 | LR: 7.78e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.2160s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0253 | max_forward: 0.1473 | max_loss: 0.0038 | max_backward: 0.0955 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 1.9569 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5248.8 | uss_MB: 4513.6 | child_rss_MB: 7321.5 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 120.0 +rank7 [2026-10-27 20:47:43,818] INFO : Step 80000/100000 rank7 | Train Loss: 15.3065 | Train Loss Unweighted: 4.8564 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4208 | Regression_cos_phi: 0.0455 | Regression_energy: 0.6415 | Regression_eta: 0.0773 | Regression_pt: 0.6092 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.6398 | physical_nopu: 4.4406 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3279 | Classification_binary: 3.2743 | Regression_cos_phi: 1.8766 | Regression_energy: 2.9684 | Regression_eta: 0.7637 | Regression_pt: 4.1590 | Regression_sin_phi: 1.3711 | LR: 7.44e-05 | DataLoad Time: 0.0092s | Model Forward Time: 3.0632s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0010 | max_forward: 0.0840 | max_loss: 0.0039 | max_backward: 0.0916 | max_optimizer_step: 0.2630 | max_tensorboard_logging: 1.7339 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5238.8 | uss_MB: 4942.0 | child_rss_MB: 7231.1 | allocated_MB: 393.8 | reserved_MB: 12330.0 | inactive_split_MB: 141.7 +rank7 [2026-10-27 20:47:44,748] INFO : Running validation on rank7 +rank7 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 20:47:44,749] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 20:49:38,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 20:49:38,983] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 20:50:04,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 20:50:05,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 20:50:05,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 20:50:06,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 20:50:06,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 20:50:07,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 20:50:07,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 20:50:08,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 20:50:18,283] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 20:50:18,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 20:50:18,743] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 20:50:33,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 20:50:34,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 20:50:34,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 20:50:35,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 20:50:35,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 20:50:36,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 20:50:36,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 20:50:37,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 20:50:47,100] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 20:50:47,267] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 20:50:47,675] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 20:51:02,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 20:51:03,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 20:51:03,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 20:51:03,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 20:51:04,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 20:51:04,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 20:51:05,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 20:51:05,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_80000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 20:51:15,720] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-27 20:54:47,405] INFO : Step 81000/100000 rank7 | Train Loss: 10.7072 | Train Loss Unweighted: 3.6241 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.9080 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2710 | Regression_eta: 0.1078 | Regression_pt: 0.2269 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6413 | physical_nopu: 4.4395 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3642 | Classification_binary: 2.9225 | Regression_cos_phi: 1.7951 | Regression_energy: 1.3366 | Regression_eta: 1.1946 | Regression_pt: 1.6703 | Regression_sin_phi: 1.6457 | LR: 7.11e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.4725s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0033 | max_forward: 0.1984 | max_loss: 0.0038 | max_backward: 0.3244 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 1.8991 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 7314.5 | uss_MB: 7027.3 | child_rss_MB: 7215.1 | allocated_MB: 408.0 | reserved_MB: 11650.0 | inactive_split_MB: 261.6 +rank7 [2026-10-27 20:58:04,709] INFO : Step 82000/100000 rank7 | Train Loss: 12.9094 | Train Loss Unweighted: 4.6043 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5247 | Regression_cos_phi: 0.0417 | Regression_energy: 0.4765 | Regression_eta: 0.0603 | Regression_pt: 0.4449 | Regression_sin_phi: 0.0402 | Group Train Loss Unweighted (window mean): physical_pu: 3.6409 | physical_nopu: 4.4420 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3181 | Classification_binary: 3.5393 | Regression_cos_phi: 1.7354 | Regression_energy: 1.5857 | Regression_eta: 0.8922 | Regression_pt: 2.1314 | Regression_sin_phi: 1.3208 | LR: 6.80e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.1712s | Phase Times: max_pre_step_barrier: 0.0156 | max_batch_to_device: 0.0008 | max_forward: 0.1472 | max_loss: 0.0038 | max_backward: 0.1553 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 1.8498 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5342.3 | uss_MB: 5059.9 | child_rss_MB: 7159.0 | allocated_MB: 393.8 | reserved_MB: 11650.0 | inactive_split_MB: 92.9 +rank7 [2026-10-27 21:01:20,553] INFO : Step 83000/100000 rank7 | Train Loss: 11.2556 | Train Loss Unweighted: 4.2618 | Train Loss Components: Classification: 0.0094 | Classification_binary: 3.5167 | Regression_cos_phi: 0.0501 | Regression_energy: 0.3127 | Regression_eta: 0.0669 | Regression_pt: 0.2586 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6408 | physical_nopu: 4.4397 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2079 | Classification_binary: 3.5356 | Regression_cos_phi: 2.0546 | Regression_energy: 1.4483 | Regression_eta: 0.8709 | Regression_pt: 1.5554 | Regression_sin_phi: 1.8421 | LR: 6.51e-05 | DataLoad Time: 0.0012s | Model Forward Time: 2.5758s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0008 | max_forward: 0.1683 | max_loss: 0.0037 | max_backward: 0.1023 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 1.3519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 5336.3 | uss_MB: 5061.6 | child_rss_MB: 7095.7 | allocated_MB: 393.2 | reserved_MB: 11650.0 | inactive_split_MB: 95.2 +rank7 [2026-10-27 21:04:33,664] INFO : Step 84000/100000 rank7 | Train Loss: 11.5964 | Train Loss Unweighted: 4.2611 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4346 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3486 | Regression_eta: 0.0638 | Regression_pt: 0.3055 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6366 | physical_nopu: 4.4419 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3746 | Classification_binary: 3.5005 | Regression_cos_phi: 1.5792 | Regression_energy: 1.6200 | Regression_eta: 0.6976 | Regression_pt: 1.9615 | Regression_sin_phi: 1.5829 | LR: 6.23e-05 | DataLoad Time: 0.0759s | Model Forward Time: 4.8598s | Phase Times: max_pre_step_barrier: 0.2933 | max_batch_to_device: 0.0008 | max_forward: 0.1129 | max_loss: 0.0110 | max_backward: 0.1312 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 3.4426 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5330.3 | uss_MB: 5063.1 | child_rss_MB: 7029.9 | allocated_MB: 393.9 | reserved_MB: 11650.0 | inactive_split_MB: 95.6 +rank7 [2026-10-27 21:07:28,408] INFO : Step 85000/100000 rank7 | Train Loss: 14.1090 | Train Loss Unweighted: 4.6641 | Train Loss Components: Classification: 0.0173 | Classification_binary: 3.3775 | Regression_cos_phi: 0.0425 | Regression_energy: 0.5754 | Regression_eta: 0.0615 | Regression_pt: 0.5479 | Regression_sin_phi: 0.0420 | Group Train Loss Unweighted (window mean): physical_pu: 3.6325 | physical_nopu: 4.4336 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3593 | Classification_binary: 3.4835 | Regression_cos_phi: 1.4933 | Regression_energy: 2.5809 | Regression_eta: 0.7341 | Regression_pt: 3.6783 | Regression_sin_phi: 1.4584 | LR: 5.96e-05 | DataLoad Time: 0.0020s | Model Forward Time: 2.5391s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0009 | max_forward: 0.0893 | max_loss: 0.0496 | max_backward: 0.1488 | max_optimizer_step: 0.1769 | max_tensorboard_logging: 1.3583 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 5323.1 | uss_MB: 5063.4 | child_rss_MB: 7037.3 | allocated_MB: 394.0 | reserved_MB: 11698.0 | inactive_split_MB: 93.1 +rank7 [2026-10-27 21:10:26,799] INFO : Step 86000/100000 rank7 | Train Loss: 11.7921 | Train Loss Unweighted: 4.3412 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.5823 | Regression_cos_phi: 0.0537 | Regression_energy: 0.3118 | Regression_eta: 0.0750 | Regression_pt: 0.2521 | Regression_sin_phi: 0.0550 | Group Train Loss Unweighted (window mean): physical_pu: 3.6404 | physical_nopu: 4.4337 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2427 | Classification_binary: 3.5070 | Regression_cos_phi: 1.5420 | Regression_energy: 1.5009 | Regression_eta: 1.0022 | Regression_pt: 1.8451 | Regression_sin_phi: 1.6383 | LR: 5.71e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.4341s | Phase Times: max_pre_step_barrier: 0.0233 | max_batch_to_device: 0.0898 | max_forward: 0.1824 | max_loss: 0.0038 | max_backward: 0.0938 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.9995 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1323 | Memory: rss_MB: 5319.1 | uss_MB: 4616.7 | child_rss_MB: 7006.7 | allocated_MB: 393.2 | reserved_MB: 11704.0 | inactive_split_MB: 96.0 +rank7 [2026-10-27 21:13:31,178] INFO : Step 87000/100000 rank7 | Train Loss: 12.0054 | Train Loss Unweighted: 4.3232 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.3964 | Regression_cos_phi: 0.0444 | Regression_energy: 0.4046 | Regression_eta: 0.0620 | Regression_pt: 0.3567 | Regression_sin_phi: 0.0418 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3454 | Classification_binary: 3.3341 | Regression_cos_phi: 1.7481 | Regression_energy: 3.2394 | Regression_eta: 0.9156 | Regression_pt: 4.2745 | Regression_sin_phi: 1.4687 | LR: 5.48e-05 | DataLoad Time: 0.0010s | Model Forward Time: 2.3333s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.1290 | max_loss: 0.0039 | max_backward: 0.0975 | max_optimizer_step: 0.1784 | max_tensorboard_logging: 1.1620 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5310.9 | uss_MB: 5064.7 | child_rss_MB: 7011.5 | allocated_MB: 394.7 | reserved_MB: 11704.0 | inactive_split_MB: 94.3 +rank7 [2026-10-27 21:16:33,434] INFO : Step 88000/100000 rank7 | Train Loss: 14.2663 | Train Loss Unweighted: 4.6673 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.4533 | Regression_cos_phi: 0.0523 | Regression_energy: 0.5342 | Regression_eta: 0.0666 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0508 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4372 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2789 | Classification_binary: 3.6769 | Regression_cos_phi: 2.4079 | Regression_energy: 2.2809 | Regression_eta: 0.9219 | Regression_pt: 2.8263 | Regression_sin_phi: 1.8485 | LR: 5.26e-05 | DataLoad Time: 0.0027s | Model Forward Time: 2.3213s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0008 | max_forward: 0.0997 | max_loss: 0.0039 | max_backward: 0.1038 | max_optimizer_step: 0.2517 | max_tensorboard_logging: 1.0175 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1083 | Memory: rss_MB: 5309.9 | uss_MB: 5065.2 | child_rss_MB: 7012.4 | allocated_MB: 393.7 | reserved_MB: 11704.0 | inactive_split_MB: 103.2 +rank7 [2026-10-27 21:19:30,808] INFO : Step 89000/100000 rank7 | Train Loss: 11.9885 | Train Loss Unweighted: 4.3122 | Train Loss Components: Classification: 0.0138 | Classification_binary: 3.4837 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3438 | Regression_eta: 0.0687 | Regression_pt: 0.2984 | Regression_sin_phi: 0.0554 | Group Train Loss Unweighted (window mean): physical_pu: 3.6329 | physical_nopu: 4.4374 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3016 | Classification_binary: 3.5767 | Regression_cos_phi: 1.5867 | Regression_energy: 1.4258 | Regression_eta: 0.7111 | Regression_pt: 1.8599 | Regression_sin_phi: 1.3816 | LR: 5.06e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.2337s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0008 | max_forward: 0.0976 | max_loss: 0.0053 | max_backward: 0.1242 | max_optimizer_step: 0.1785 | max_tensorboard_logging: 1.0326 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 5307.5 | uss_MB: 5066.4 | child_rss_MB: 7006.7 | allocated_MB: 395.0 | reserved_MB: 13422.0 | inactive_split_MB: 109.9 +rank7 [2026-10-27 21:22:22,522] INFO : Step 90000/100000 rank7 | Train Loss: 10.5888 | Train Loss Unweighted: 3.6256 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9282 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2656 | Regression_eta: 0.1044 | Regression_pt: 0.2178 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6354 | physical_nopu: 4.4354 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3276 | Classification_binary: 2.9590 | Regression_cos_phi: 1.7791 | Regression_energy: 1.2903 | Regression_eta: 1.1312 | Regression_pt: 1.5580 | Regression_sin_phi: 1.6021 | LR: 4.88e-05 | DataLoad Time: 0.0020s | Model Forward Time: 4.1185s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0033 | max_forward: 0.1953 | max_loss: 0.0645 | max_backward: 0.8258 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.9975 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5307.5 | uss_MB: 5074.7 | child_rss_MB: 7021.2 | allocated_MB: 404.6 | reserved_MB: 13422.0 | inactive_split_MB: 208.4 +rank7 [2026-10-27 21:22:23,511] INFO : Running validation on rank7 +rank7 [2026-10-27 21:22:23,511] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 21:22:23,511] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 21:22:23,511] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 21:22:23,512] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 21:24:23,625] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 21:24:24,118] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 21:24:48,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 21:24:49,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 21:24:49,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 21:24:50,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 21:24:50,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 21:24:51,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 21:24:51,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 21:24:52,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 21:25:02,147] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 21:25:02,472] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 21:25:02,871] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 21:25:18,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 21:25:19,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 21:25:19,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 21:25:19,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 21:25:20,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 21:25:20,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 21:25:21,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 21:25:21,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 21:25:31,786] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 21:25:32,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 21:25:32,493] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 21:25:47,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 21:25:48,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 21:25:48,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 21:25:48,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 21:25:49,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 21:25:49,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 21:25:50,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 21:25:50,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_90000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 21:26:00,903] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 21:29:48,334] INFO : Step 91000/100000 rank7 | Train Loss: 11.9808 | Train Loss Unweighted: 4.3520 | Train Loss Components: Classification: 0.0128 | Classification_binary: 3.5260 | Regression_cos_phi: 0.0522 | Regression_energy: 0.3480 | Regression_eta: 0.0723 | Regression_pt: 0.2901 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4346 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2958 | Classification_binary: 3.4410 | Regression_cos_phi: 1.9689 | Regression_energy: 2.4222 | Regression_eta: 0.9375 | Regression_pt: 2.8773 | Regression_sin_phi: 2.3504 | LR: 4.71e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.6016s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1743 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.3267 | max_tensorboard_logging: 2.0733 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0716 | Memory: rss_MB: 7308.2 | uss_MB: 7066.1 | child_rss_MB: 7044.9 | allocated_MB: 393.7 | reserved_MB: 11392.0 | inactive_split_MB: 89.1 +rank7 [2026-10-27 21:33:08,656] INFO : Step 92000/100000 rank7 | Train Loss: 14.4974 | Train Loss Unweighted: 4.7629 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4460 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5917 | Regression_eta: 0.0739 | Regression_pt: 0.5467 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6297 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3362 | Classification_binary: 3.3891 | Regression_cos_phi: 1.7629 | Regression_energy: 2.4026 | Regression_eta: 0.7230 | Regression_pt: 3.1805 | Regression_sin_phi: 1.5250 | LR: 4.57e-05 | DataLoad Time: 0.0020s | Model Forward Time: 2.6194s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1620 | max_loss: 0.0037 | max_backward: 0.2478 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 1.1676 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0595 | Memory: rss_MB: 7302.2 | uss_MB: 6481.6 | child_rss_MB: 6997.3 | allocated_MB: 394.0 | reserved_MB: 11392.0 | inactive_split_MB: 90.6 +rank7 [2026-10-27 21:36:11,295] INFO : Step 93000/100000 rank7 | Train Loss: 13.1041 | Train Loss Unweighted: 4.5189 | Train Loss Components: Classification: 0.0119 | Classification_binary: 3.5265 | Regression_cos_phi: 0.0575 | Regression_energy: 0.4342 | Regression_eta: 0.0649 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6346 | physical_nopu: 4.4327 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3236 | Classification_binary: 3.4972 | Regression_cos_phi: 1.7635 | Regression_energy: 1.8621 | Regression_eta: 0.8169 | Regression_pt: 2.3397 | Regression_sin_phi: 1.6849 | LR: 4.43e-05 | DataLoad Time: 0.0024s | Model Forward Time: 2.4217s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1013 | max_loss: 0.1069 | max_backward: 0.2035 | max_optimizer_step: 0.1792 | max_tensorboard_logging: 1.1596 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 5379.1 | uss_MB: 5141.2 | child_rss_MB: 6951.6 | allocated_MB: 393.6 | reserved_MB: 13294.0 | inactive_split_MB: 89.2 +rank7 [2026-10-27 21:39:11,302] INFO : Step 94000/100000 rank7 | Train Loss: 11.5177 | Train Loss Unweighted: 4.2756 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.4441 | Regression_cos_phi: 0.0413 | Regression_energy: 0.3503 | Regression_eta: 0.0686 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6308 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3999 | Classification_binary: 3.4445 | Regression_cos_phi: 1.3800 | Regression_energy: 2.3176 | Regression_eta: 0.5713 | Regression_pt: 3.1358 | Regression_sin_phi: 1.2696 | LR: 4.32e-05 | DataLoad Time: 0.0024s | Model Forward Time: 6.3160s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0191 | max_forward: 0.1721 | max_loss: 0.0188 | max_backward: 0.2028 | max_optimizer_step: 0.2895 | max_tensorboard_logging: 4.7123 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 5372.1 | uss_MB: 5157.2 | child_rss_MB: 6925.7 | allocated_MB: 394.1 | reserved_MB: 13294.0 | inactive_split_MB: 89.1 +rank7 [2026-10-27 21:42:20,694] INFO : Step 95000/100000 rank7 | Train Loss: 14.1207 | Train Loss Unweighted: 4.6477 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3920 | Regression_cos_phi: 0.0441 | Regression_energy: 0.5776 | Regression_eta: 0.0602 | Regression_pt: 0.5087 | Regression_sin_phi: 0.0499 | Group Train Loss Unweighted (window mean): physical_pu: 3.6267 | physical_nopu: 4.4321 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2839 | Classification_binary: 3.5357 | Regression_cos_phi: 1.6196 | Regression_energy: 2.0605 | Regression_eta: 0.7790 | Regression_pt: 2.7021 | Regression_sin_phi: 1.6608 | LR: 4.22e-05 | DataLoad Time: 0.0013s | Model Forward Time: 2.5041s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0008 | max_forward: 0.0941 | max_loss: 0.0036 | max_backward: 0.1734 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 1.2811 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5361.0 | uss_MB: 5157.8 | child_rss_MB: 6928.3 | allocated_MB: 396.0 | reserved_MB: 14294.0 | inactive_split_MB: 91.1 +rank7 [2026-10-27 21:45:16,250] INFO : Step 96000/100000 rank7 | Train Loss: 11.5036 | Train Loss Unweighted: 4.2276 | Train Loss Components: Classification: 0.0097 | Classification_binary: 3.4897 | Regression_cos_phi: 0.0538 | Regression_energy: 0.2982 | Regression_eta: 0.0802 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6278 | physical_nopu: 4.4329 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2350 | Classification_binary: 3.5144 | Regression_cos_phi: 1.8488 | Regression_energy: 1.4736 | Regression_eta: 1.1903 | Regression_pt: 1.7041 | Regression_sin_phi: 1.9820 | LR: 4.14e-05 | DataLoad Time: 0.0022s | Model Forward Time: 2.3869s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0007 | max_forward: 0.0991 | max_loss: 0.0037 | max_backward: 0.1000 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5357.0 | uss_MB: 5158.4 | child_rss_MB: 6899.7 | allocated_MB: 393.0 | reserved_MB: 14294.0 | inactive_split_MB: 95.3 +rank7 [2026-10-27 21:48:34,573] INFO : Step 97000/100000 rank7 | Train Loss: 11.8793 | Train Loss Unweighted: 4.3044 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.4420 | Regression_cos_phi: 0.0454 | Regression_energy: 0.3663 | Regression_eta: 0.0711 | Regression_pt: 0.3156 | Regression_sin_phi: 0.0468 | Group Train Loss Unweighted (window mean): physical_pu: 3.6277 | physical_nopu: 4.4255 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.4528 | Classification_binary: 3.4465 | Regression_cos_phi: 1.5010 | Regression_energy: 1.4399 | Regression_eta: 0.7249 | Regression_pt: 1.9510 | Regression_sin_phi: 1.5306 | LR: 4.08e-05 | DataLoad Time: 0.0022s | Model Forward Time: 2.5911s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1192 | max_loss: 0.0048 | max_backward: 0.1852 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.3131 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 5356.0 | uss_MB: 5157.9 | child_rss_MB: 6903.2 | allocated_MB: 394.7 | reserved_MB: 14294.0 | inactive_split_MB: 98.1 +rank7 [2026-10-27 21:51:32,851] INFO : Step 98000/100000 rank7 | Train Loss: 12.9367 | Train Loss Unweighted: 4.4840 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.4015 | Regression_cos_phi: 0.0441 | Regression_energy: 0.4811 | Regression_eta: 0.0611 | Regression_pt: 0.4378 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4275 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3278 | Classification_binary: 3.4859 | Regression_cos_phi: 1.5052 | Regression_energy: 2.6008 | Regression_eta: 0.6282 | Regression_pt: 3.6425 | Regression_sin_phi: 1.7091 | LR: 4.04e-05 | DataLoad Time: 0.0013s | Model Forward Time: 2.8948s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1318 | max_loss: 0.0038 | max_backward: 0.1127 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.6868 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 5362.0 | uss_MB: 5160.3 | child_rss_MB: 6899.7 | allocated_MB: 394.5 | reserved_MB: 14294.0 | inactive_split_MB: 82.0 +rank7 [2026-10-27 21:54:35,739] INFO : Step 99000/100000 rank7 | Train Loss: 11.6408 | Train Loss Unweighted: 4.3204 | Train Loss Components: Classification: 0.0127 | Classification_binary: 3.5557 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3117 | Regression_eta: 0.0641 | Regression_pt: 0.2727 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 3.6292 | physical_nopu: 4.4284 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.2487 | Classification_binary: 3.5250 | Regression_cos_phi: 1.8525 | Regression_energy: 1.3008 | Regression_eta: 0.6474 | Regression_pt: 1.5366 | Regression_sin_phi: 1.6637 | LR: 4.01e-05 | DataLoad Time: 0.0077s | Model Forward Time: 3.0811s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0008 | max_forward: 0.1731 | max_loss: 0.0038 | max_backward: 0.1092 | max_optimizer_step: 0.1829 | max_tensorboard_logging: 1.8331 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 5358.0 | uss_MB: 5160.5 | child_rss_MB: 6886.0 | allocated_MB: 394.3 | reserved_MB: 14294.0 | inactive_split_MB: 113.7 +rank7 [2026-10-27 21:57:56,509] INFO : Step 100000/100000 rank7 | Train Loss: 10.8909 | Train Loss Unweighted: 3.6545 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.9133 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2833 | Regression_eta: 0.1041 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6336 | physical_nopu: 4.4311 | Task Weights: Classification: 22.2936 | Classification_binary: 1.0000 | Regression_cos_phi: 36.1697 | Regression_energy: 4.8476 | Regression_eta: 11.0915 | Regression_pt: 7.1219 | Regression_sin_phi: 34.1452 | Task Weighted Losses: Classification: 0.3387 | Classification_binary: 2.8940 | Regression_cos_phi: 1.6784 | Regression_energy: 1.4339 | Regression_eta: 1.1851 | Regression_pt: 1.8562 | Regression_sin_phi: 1.6686 | LR: 4.00e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.7459s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0030 | max_forward: 0.2088 | max_loss: 0.0393 | max_backward: 0.3031 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 1.9276 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 5350.9 | uss_MB: 5163.3 | child_rss_MB: 6834.2 | allocated_MB: 405.7 | reserved_MB: 14294.0 | inactive_split_MB: 165.9 +rank7 [2026-10-27 21:57:57,460] INFO : Running validation on rank7 +rank7 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 21:57:57,461] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 21:59:51,634] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 21:59:52,048] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 22:00:18,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 22:00:19,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 22:00:19,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 22:00:20,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 22:00:20,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 22:00:20,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 22:00:21,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 22:00:21,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 22:00:31,848] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 22:00:32,062] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 22:00:32,465] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 22:00:47,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 22:00:48,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 22:00:48,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 22:00:49,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 22:00:49,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 22:00:50,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 22:00:50,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 22:00:51,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 22:01:01,265] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 22:01:01,522] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 22:01:01,928] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 22:01:17,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 22:01:17,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 22:01:18,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 22:01:18,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 22:01:19,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 22:01:19,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 22:01:20,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 22:01:20,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/attention_seed12345_20261027_160940_955225/preds_step_100000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 22:01:30,466] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 22:01:30,973] INFO : Training completed. Total time on device 7: 350.238min diff --git a/cms/v3.2.0/comparison_metrics/comparison.json b/cms/v3.2.0/comparison_metrics/comparison.json new file mode 100644 index 0000000000000000000000000000000000000000..b6fa499b77ac73807ccdc4bf3ff9b8d05ae145a3 --- /dev/null +++ b/cms/v3.2.0/comparison_metrics/comparison.json @@ -0,0 +1,179 @@ +{ + "events": 1536, + "particle_metrics": { + "matching/f1": { + "reference": 0.6331278372676371, + "attention": 0.6256042593083398, + "heptv2": 0.6287424904733289 + }, + "matching/efficiency": { + "reference": 0.647983952539522, + "attention": 0.6433049585743672, + "heptv2": 0.6462812397004171 + }, + "matching/purity": { + "reference": 0.618937656877149, + "attention": 0.6088515542108647, + "heptv2": 0.6121305217878983 + }, + "matched/pid_accuracy": { + "reference": 0.8277123432837339, + "attention": 0.8418942487212864, + "heptv2": 0.8432160439942292 + }, + "matched/pt_relative_abs_mean": { + "reference": 0.10350282075128193, + "attention": 0.10589761266584412, + "heptv2": 0.10495726544972905 + }, + "event/met_abs_error": { + "reference": 21.048562037536346, + "attention": 23.567323698539713, + "heptv2": 23.019306580611147 + }, + "event/energy_relative_abs_error": { + "reference": 0.023899524677626804, + "attention": 0.02217946297894055, + "heptv2": 0.02399052382306081 + } + }, + "paired_bootstrap_new_minus_reference_95ci": { + "matching/f1": { + "attention": [ + -0.007766242036837112, + -0.007291269540749587 + ], + "heptv2": [ + -0.004617316740288202, + -0.004157413597673207 + ] + }, + "matching/efficiency": { + "attention": [ + -0.00492972320236226, + -0.004425835388240781 + ], + "heptv2": [ + -0.0019458644248883856, + -0.0014546529836558578 + ] + }, + "matching/purity": { + "attention": [ + -0.010336279779517196, + -0.00984909916667189 + ], + "heptv2": [ + -0.007044944021339039, + -0.00657487415635101 + ] + }, + "matched/pid_accuracy": { + "attention": [ + 0.013774041339399076, + 0.014564463659544476 + ], + "heptv2": [ + 0.015086512042998101, + 0.015894510490218283 + ] + }, + "matched/pt_relative_abs_mean": { + "attention": [ + 0.0022852865471054496, + 0.0025035548996949443 + ], + "heptv2": [ + 0.001350010359437211, + 0.0015614038135931339 + ] + }, + "event/met_abs_error": { + "attention": [ + 0.7846455705651452, + 4.933147771107289 + ], + "heptv2": [ + 0.9368793968426824, + 2.9494956021825933 + ] + }, + "event/energy_relative_abs_error": { + "attention": [ + -0.0020513128110047204, + -0.001373732904041496 + ], + "heptv2": [ + -0.00023577788646995712, + 0.00042125550820724836 + ] + } + }, + "unique_events": 1530, + "reference_precision": "Published FP32 ONNX export, single unpadded event per inference", + "current_precision": "Saved LUMI predictions from bfloat16 autocast", + "jet_definition": "anti-kt R=0.4, pt>3 GeV, one-to-one angular matching deltaR<0.1; response is predicted/target pt", + "method": "2000 paired unique-event cluster bootstrap draws, stratified by process; duplicate entries grouped; aggregate sums/counts as in validation_metrics", + "jets": { + "cms_pf_qcd": { + "reference": { + "median": 1.0279237433407529, + "iqr": 0.38068409177597384, + "angular_recall": 0.19813143933332514, + "angular_precision": 0.19038381926784553 + }, + "attention": { + "median": 1.048952555174973, + "iqr": 0.37771686552257944, + "angular_recall": 0.19697435960560813, + "angular_precision": 0.19026443450965472 + }, + "heptv2": { + "median": 1.0255487525366904, + "iqr": 0.36232087003909597, + "angular_recall": 0.18925639163455976, + "angular_precision": 0.18819079792898322 + } + }, + "cms_pf_ttbar": { + "reference": { + "median": 1.0292553553056807, + "iqr": 0.365230819372485, + "angular_recall": 0.20795335992635777, + "angular_precision": 0.19975241688281065 + }, + "attention": { + "median": 1.0485542740717433, + "iqr": 0.35525660407754733, + "angular_recall": 0.20708192697146363, + "angular_precision": 0.1998791626683726 + }, + "heptv2": { + "median": 1.0266958929906298, + "iqr": 0.3431011058542559, + "angular_recall": 0.19310217858238723, + "angular_precision": 0.19242435361170226 + } + }, + "cms_pf_ztt": { + "reference": { + "median": 1.0360193937900064, + "iqr": 0.40236522848229617, + "angular_recall": 0.19215487256928543, + "angular_precision": 0.18457660571183981 + }, + "attention": { + "median": 1.050952585178886, + "iqr": 0.38454960721696474, + "angular_recall": 0.18995160232209832, + "angular_precision": 0.18345048534404437 + }, + "heptv2": { + "median": 1.0255522306353952, + "iqr": 0.3843649145070648, + "angular_recall": 0.17912091992721782, + "angular_precision": 0.17831969637223974 + } + } + } +} diff --git a/cms/v3.2.0/comparison_metrics/provenance.json b/cms/v3.2.0/comparison_metrics/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..e0e54c393a6e99c60b9c1317d5338de55c2f7371 --- /dev/null +++ b/cms/v3.2.0/comparison_metrics/provenance.json @@ -0,0 +1,1184 @@ +{ + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "reference_checkpoint_sha256": "110fcbe0fa3688f30413c470f1b32bc83e1ec6867f9ed4bc6b78890f46e4a779", + "reference_parameters": 5133603, + "onnx_matching_initializers": [ + [ + "nn0_id.0.0.bias", + "nn0_id.0.0.bias" + ], + [ + "nn0_id.0.2.weight", + "nn0_id.0.2.weight" + ], + [ + "nn0_id.0.2.bias", + "nn0_id.0.2.bias" + ], + [ + "nn0_id.0.4.bias", + "nn0_id.0.4.bias" + ], + [ + "nn0_id.1.0.bias", + "nn0_id.1.0.bias" + ], + [ + "nn0_id.1.2.weight", + "nn0_id.1.2.weight" + ], + [ + "nn0_id.1.2.bias", + "nn0_id.1.2.bias" + ], + [ + "nn0_id.1.4.bias", + "nn0_id.1.4.bias" + ], + [ + "nn0_id.2.0.bias", + "nn0_id.2.0.bias" + ], + [ + "nn0_id.2.2.weight", + "nn0_id.2.2.weight" + ], + [ + "nn0_id.2.2.bias", + "nn0_id.2.2.bias" + ], + [ + "nn0_id.2.4.bias", + "nn0_id.2.4.bias" + ], + [ + "nn0_id.3.0.bias", + "nn0_id.3.0.bias" + ], + [ + "nn0_id.3.2.weight", + "nn0_id.3.2.weight" + ], + [ + "nn0_id.3.2.bias", + "nn0_id.3.2.bias" + ], + [ + "nn0_id.3.4.bias", + "nn0_id.3.4.bias" + ], + [ + "nn0_id.4.0.bias", + "nn0_id.4.0.bias" + ], + [ + "nn0_id.4.2.weight", + "nn0_id.4.2.weight" + ], + [ + "nn0_id.4.2.bias", + "nn0_id.4.2.bias" + ], + [ + "nn0_id.4.4.bias", + "nn0_id.4.4.bias" + ], + [ + "nn0_id.5.0.bias", + "nn0_id.5.0.bias" + ], + [ + "nn0_id.5.2.weight", + "nn0_id.5.2.weight" + ], + [ + "nn0_id.5.2.bias", + "nn0_id.5.2.bias" + ], + [ + "nn0_id.5.4.bias", + "nn0_id.5.4.bias" + ], + [ + "nn0_id.6.0.bias", + "nn0_id.6.0.bias" + ], + [ + "nn0_id.6.2.weight", + "nn0_id.6.2.weight" + ], + [ + "nn0_id.6.2.bias", + "nn0_id.6.2.bias" + ], + [ + "nn0_id.6.4.bias", + "nn0_id.6.4.bias" + ], + [ + "nn0_id.7.0.bias", + "nn0_id.7.0.bias" + ], + [ + "nn0_id.7.2.weight", + "nn0_id.7.2.weight" + ], + [ + "nn0_id.7.2.bias", + "nn0_id.7.2.bias" + ], + [ + "nn0_id.7.4.bias", + "nn0_id.7.4.bias" + ], + [ + "nn0_reg.0.0.bias", + "nn0_reg.0.0.bias" + ], + [ + "nn0_reg.0.2.weight", + "nn0_reg.0.2.weight" + ], + [ + "nn0_reg.0.2.bias", + "nn0_reg.0.2.bias" + ], + [ + "nn0_reg.0.4.bias", + "nn0_reg.0.4.bias" + ], + [ + "nn0_reg.1.0.bias", + "nn0_reg.1.0.bias" + ], + [ + "nn0_reg.1.2.weight", + "nn0_reg.1.2.weight" + ], + [ + "nn0_reg.1.2.bias", + "nn0_reg.1.2.bias" + ], + [ + "nn0_reg.1.4.bias", + "nn0_reg.1.4.bias" + ], + [ + "nn0_reg.2.0.bias", + "nn0_reg.2.0.bias" + ], + [ + "nn0_reg.2.2.weight", + "nn0_reg.2.2.weight" + ], + [ + "nn0_reg.2.2.bias", + "nn0_reg.2.2.bias" + ], + [ + "nn0_reg.2.4.bias", + "nn0_reg.2.4.bias" + ], + [ + "nn0_reg.3.0.bias", + "nn0_reg.3.0.bias" + ], + [ + "nn0_reg.3.2.weight", + "nn0_reg.3.2.weight" + ], + [ + "nn0_reg.3.2.bias", + "nn0_reg.3.2.bias" + ], + [ + "nn0_reg.3.4.bias", + "nn0_reg.3.4.bias" + ], + [ + "nn0_reg.4.0.bias", + "nn0_reg.4.0.bias" + ], + [ + "nn0_reg.4.2.weight", + "nn0_reg.4.2.weight" + ], + [ + "nn0_reg.4.2.bias", + "nn0_reg.4.2.bias" + ], + [ + "nn0_reg.4.4.bias", + "nn0_reg.4.4.bias" + ], + [ + "nn0_reg.5.0.bias", + "nn0_reg.5.0.bias" + ], + [ + "nn0_reg.5.2.weight", + "nn0_reg.5.2.weight" + ], + [ + "nn0_reg.5.2.bias", + "nn0_reg.5.2.bias" + ], + [ + "nn0_reg.5.4.bias", + "nn0_reg.5.4.bias" + ], + [ + "nn0_reg.6.0.bias", + "nn0_reg.6.0.bias" + ], + [ + "nn0_reg.6.2.weight", + "nn0_reg.6.2.weight" + ], + [ + "nn0_reg.6.2.bias", + "nn0_reg.6.2.bias" + ], + [ + "nn0_reg.6.4.bias", + "nn0_reg.6.4.bias" + ], + [ + "nn0_reg.7.0.bias", + "nn0_reg.7.0.bias" + ], + [ + "nn0_reg.7.2.weight", + "nn0_reg.7.2.weight" + ], + [ + "nn0_reg.7.2.bias", + "nn0_reg.7.2.bias" + ], + [ + "nn0_reg.7.4.bias", + "nn0_reg.7.4.bias" + ], + [ + "conv_id.0.mha.out_proj.bias", + "conv_id.0.mha.out_proj.bias" + ], + [ + "conv_id.0.norm0.weight", + "conv_id.0.norm0.weight" + ], + [ + "conv_id.0.norm0.bias", + "conv_id.0.norm0.bias" + ], + [ + "conv_id.0.norm1.weight", + "conv_id.0.norm1.weight" + ], + [ + "conv_id.0.norm1.bias", + "conv_id.0.norm1.bias" + ], + [ + "conv_id.0.seq.0.bias", + "conv_id.0.seq.0.bias" + ], + [ + "conv_id.0.seq.2.bias", + "conv_id.0.seq.2.bias" + ], + [ + "conv_id.1.mha.out_proj.bias", + "conv_id.1.mha.out_proj.bias" + ], + [ + "conv_id.1.norm0.weight", + "conv_id.1.norm0.weight" + ], + [ + "conv_id.1.norm0.bias", + "conv_id.1.norm0.bias" + ], + [ + "conv_id.1.norm1.weight", + "conv_id.1.norm1.weight" + ], + [ + "conv_id.1.norm1.bias", + "conv_id.1.norm1.bias" + ], + [ + "conv_id.1.seq.0.bias", + "conv_id.1.seq.0.bias" + ], + [ + "conv_id.1.seq.2.bias", + "conv_id.1.seq.2.bias" + ], + [ + "conv_id.2.mha.out_proj.bias", + "conv_id.2.mha.out_proj.bias" + ], + [ + "conv_id.2.norm0.weight", + "conv_id.2.norm0.weight" + ], + [ + "conv_id.2.norm0.bias", + "conv_id.2.norm0.bias" + ], + [ + "conv_id.2.norm1.weight", + "conv_id.2.norm1.weight" + ], + [ + "conv_id.2.norm1.bias", + "conv_id.2.norm1.bias" + ], + [ + "conv_id.2.seq.0.bias", + "conv_id.2.seq.0.bias" + ], + [ + "conv_id.2.seq.2.bias", + "conv_id.2.seq.2.bias" + ], + [ + "conv_reg.0.mha.out_proj.bias", + "conv_reg.0.mha.out_proj.bias" + ], + [ + "conv_reg.0.norm0.weight", + "conv_reg.0.norm0.weight" + ], + [ + "conv_reg.0.norm0.bias", + "conv_reg.0.norm0.bias" + ], + [ + "conv_reg.0.norm1.weight", + "conv_reg.0.norm1.weight" + ], + [ + "conv_reg.0.norm1.bias", + "conv_reg.0.norm1.bias" + ], + [ + "conv_reg.0.seq.0.bias", + "conv_reg.0.seq.0.bias" + ], + [ + "conv_reg.0.seq.2.bias", + "conv_reg.0.seq.2.bias" + ], + [ + "conv_reg.1.mha.out_proj.bias", + "conv_reg.1.mha.out_proj.bias" + ], + [ + "conv_reg.1.norm0.weight", + "conv_reg.1.norm0.weight" + ], + [ + "conv_reg.1.norm0.bias", + "conv_reg.1.norm0.bias" + ], + [ + "conv_reg.1.norm1.weight", + "conv_reg.1.norm1.weight" + ], + [ + "conv_reg.1.norm1.bias", + "conv_reg.1.norm1.bias" + ], + [ + "conv_reg.1.seq.0.bias", + "conv_reg.1.seq.0.bias" + ], + [ + "conv_reg.1.seq.2.bias", + "conv_reg.1.seq.2.bias" + ], + [ + "conv_reg.2.mha.out_proj.bias", + "conv_reg.2.mha.out_proj.bias" + ], + [ + "conv_reg.2.norm0.weight", + "conv_reg.2.norm0.weight" + ], + [ + "conv_reg.2.norm0.bias", + "conv_reg.2.norm0.bias" + ], + [ + "conv_reg.2.norm1.weight", + "conv_reg.2.norm1.weight" + ], + [ + "conv_reg.2.norm1.bias", + "conv_reg.2.norm1.bias" + ], + [ + "conv_reg.2.seq.0.bias", + "conv_reg.2.seq.0.bias" + ], + [ + "conv_reg.2.seq.2.bias", + "conv_reg.2.seq.2.bias" + ], + [ + "nn_binary_particle.0.bias", + "nn_binary_particle.0.bias" + ], + [ + "nn_binary_particle.2.weight", + "nn_binary_particle.2.weight" + ], + [ + "nn_binary_particle.2.bias", + "nn_binary_particle.2.bias" + ], + [ + "nn_binary_particle.4.bias", + "nn_binary_particle.4.bias" + ], + [ + "nn_pid.0.bias", + "nn_pid.0.bias" + ], + [ + "nn_pid.2.weight", + "nn_pid.2.weight" + ], + [ + "nn_pid.2.bias", + "nn_pid.2.bias" + ], + [ + "nn_pid.4.bias", + "nn_pid.4.bias" + ], + [ + "nn_pu.0.bias", + "nn_pu.0.bias" + ], + [ + "nn_pu.2.weight", + "nn_pu.2.weight" + ], + [ + "nn_pu.2.bias", + "nn_pu.2.bias" + ], + [ + "nn_pu.4.bias", + "nn_pu.4.bias" + ], + [ + "nn_pt.nn.0.0.bias", + "nn_pt.nn.0.0.bias" + ], + [ + "nn_pt.nn.0.2.weight", + "nn_pt.nn.0.2.weight" + ], + [ + "nn_pt.nn.0.2.bias", + "nn_pt.nn.0.2.bias" + ], + [ + "nn_pt.nn.0.4.bias", + "nn_pt.nn.0.4.bias" + ], + [ + "nn_pt.nn.1.0.bias", + "nn_pt.nn.1.0.bias" + ], + [ + "nn_pt.nn.1.2.weight", + "nn_pt.nn.1.2.weight" + ], + [ + "nn_pt.nn.1.2.bias", + "nn_pt.nn.1.2.bias" + ], + [ + "nn_pt.nn.1.4.bias", + "nn_pt.nn.1.4.bias" + ], + [ + "nn_pt.nn.2.0.bias", + "nn_pt.nn.2.0.bias" + ], + [ + "nn_pt.nn.2.2.weight", + "nn_pt.nn.2.2.weight" + ], + [ + "nn_pt.nn.2.2.bias", + "nn_pt.nn.2.2.bias" + ], + [ + "nn_pt.nn.2.4.bias", + "nn_pt.nn.2.4.bias" + ], + [ + "nn_pt.nn.3.0.bias", + "nn_pt.nn.3.0.bias" + ], + [ + "nn_pt.nn.3.2.weight", + "nn_pt.nn.3.2.weight" + ], + [ + "nn_pt.nn.3.2.bias", + "nn_pt.nn.3.2.bias" + ], + [ + "nn_pt.nn.3.4.bias", + "nn_pt.nn.3.4.bias" + ], + [ + "nn_pt.nn.4.0.bias", + "nn_pt.nn.4.0.bias" + ], + [ + "nn_pt.nn.4.2.weight", + "nn_pt.nn.4.2.weight" + ], + [ + "nn_pt.nn.4.2.bias", + "nn_pt.nn.4.2.bias" + ], + [ + "nn_pt.nn.4.4.bias", + "nn_pt.nn.4.4.bias" + ], + [ + "nn_pt.nn.5.0.bias", + "nn_pt.nn.5.0.bias" + ], + [ + "nn_pt.nn.5.2.weight", + "nn_pt.nn.5.2.weight" + ], + [ + "nn_pt.nn.5.2.bias", + "nn_pt.nn.5.2.bias" + ], + [ + "nn_pt.nn.5.4.bias", + "nn_pt.nn.5.4.bias" + ], + [ + "nn_pt.nn.6.0.bias", + "nn_pt.nn.6.0.bias" + ], + [ + "nn_pt.nn.6.2.weight", + "nn_pt.nn.6.2.weight" + ], + [ + "nn_pt.nn.6.2.bias", + "nn_pt.nn.6.2.bias" + ], + [ + "nn_pt.nn.6.4.bias", + "nn_pt.nn.6.4.bias" + ], + [ + "nn_pt.nn.7.0.bias", + "nn_pt.nn.7.0.bias" + ], + [ + "nn_pt.nn.7.2.weight", + "nn_pt.nn.7.2.weight" + ], + [ + "nn_pt.nn.7.2.bias", + "nn_pt.nn.7.2.bias" + ], + [ + "nn_pt.nn.7.4.bias", + "nn_pt.nn.7.4.bias" + ], + [ + "nn_eta.nn.0.bias", + "nn_eta.nn.0.bias" + ], + [ + "nn_eta.nn.2.weight", + "nn_eta.nn.2.weight" + ], + [ + "nn_eta.nn.2.bias", + "nn_eta.nn.2.bias" + ], + [ + "nn_eta.nn.4.bias", + "nn_eta.nn.4.bias" + ], + [ + "nn_sin_phi.nn.0.bias", + "nn_sin_phi.nn.0.bias" + ], + [ + "nn_sin_phi.nn.2.weight", + "nn_sin_phi.nn.2.weight" + ], + [ + "nn_sin_phi.nn.2.bias", + "nn_sin_phi.nn.2.bias" + ], + [ + "nn_sin_phi.nn.4.bias", + "nn_sin_phi.nn.4.bias" + ], + [ + "nn_cos_phi.nn.0.bias", + "nn_cos_phi.nn.0.bias" + ], + [ + "nn_cos_phi.nn.2.weight", + "nn_cos_phi.nn.2.weight" + ], + [ + "nn_cos_phi.nn.2.bias", + "nn_cos_phi.nn.2.bias" + ], + [ + "nn_cos_phi.nn.4.bias", + "nn_cos_phi.nn.4.bias" + ], + [ + "nn_energy.nn.0.0.bias", + "nn_energy.nn.0.0.bias" + ], + [ + "nn_energy.nn.0.2.weight", + "nn_energy.nn.0.2.weight" + ], + [ + "nn_energy.nn.0.2.bias", + "nn_energy.nn.0.2.bias" + ], + [ + "nn_energy.nn.0.4.bias", + "nn_energy.nn.0.4.bias" + ], + [ + "nn_energy.nn.1.0.bias", + "nn_energy.nn.1.0.bias" + ], + [ + "nn_energy.nn.1.2.weight", + "nn_energy.nn.1.2.weight" + ], + [ + "nn_energy.nn.1.2.bias", + "nn_energy.nn.1.2.bias" + ], + [ + "nn_energy.nn.1.4.bias", + "nn_energy.nn.1.4.bias" + ], + [ + "nn_energy.nn.2.0.bias", + "nn_energy.nn.2.0.bias" + ], + [ + "nn_energy.nn.2.2.weight", + "nn_energy.nn.2.2.weight" + ], + [ + "nn_energy.nn.2.2.bias", + "nn_energy.nn.2.2.bias" + ], + [ + "nn_energy.nn.2.4.bias", + "nn_energy.nn.2.4.bias" + ], + [ + "nn_energy.nn.3.0.bias", + "nn_energy.nn.3.0.bias" + ], + [ + "nn_energy.nn.3.2.weight", + "nn_energy.nn.3.2.weight" + ], + [ + "nn_energy.nn.3.2.bias", + "nn_energy.nn.3.2.bias" + ], + [ + "nn_energy.nn.3.4.bias", + "nn_energy.nn.3.4.bias" + ], + [ + "nn_energy.nn.4.0.bias", + "nn_energy.nn.4.0.bias" + ], + [ + "nn_energy.nn.4.2.weight", + "nn_energy.nn.4.2.weight" + ], + [ + "nn_energy.nn.4.2.bias", + "nn_energy.nn.4.2.bias" + ], + [ + "nn_energy.nn.4.4.bias", + "nn_energy.nn.4.4.bias" + ], + [ + "nn_energy.nn.5.0.bias", + "nn_energy.nn.5.0.bias" + ], + [ + "nn_energy.nn.5.2.weight", + "nn_energy.nn.5.2.weight" + ], + [ + "nn_energy.nn.5.2.bias", + "nn_energy.nn.5.2.bias" + ], + [ + "nn_energy.nn.5.4.bias", + "nn_energy.nn.5.4.bias" + ], + [ + "nn_energy.nn.6.0.bias", + "nn_energy.nn.6.0.bias" + ], + [ + "nn_energy.nn.6.2.weight", + "nn_energy.nn.6.2.weight" + ], + [ + "nn_energy.nn.6.2.bias", + "nn_energy.nn.6.2.bias" + ], + [ + "nn_energy.nn.6.4.bias", + "nn_energy.nn.6.4.bias" + ], + [ + "nn_energy.nn.7.0.bias", + "nn_energy.nn.7.0.bias" + ], + [ + "nn_energy.nn.7.2.weight", + "nn_energy.nn.7.2.weight" + ], + [ + "nn_energy.nn.7.2.bias", + "nn_energy.nn.7.2.bias" + ], + [ + "nn_energy.nn.7.4.bias", + "nn_energy.nn.7.4.bias" + ], + [ + "final_norm_id.weight", + "final_norm_id.weight" + ], + [ + "final_norm_id.bias", + "final_norm_id.bias" + ], + [ + "final_norm_reg.weight", + "final_norm_reg.weight" + ], + [ + "final_norm_reg.bias", + "final_norm_reg.bias" + ], + [ + "onnx::MatMul_1647", + "nn0_id.0.0.weight (transposed)" + ], + [ + "onnx::MatMul_1648", + "nn0_id.0.4.weight (transposed)" + ], + [ + "onnx::MatMul_1649", + "nn0_id.1.0.weight (transposed)" + ], + [ + "onnx::MatMul_1650", + "nn0_id.1.4.weight (transposed)" + ], + [ + "onnx::MatMul_1651", + "nn0_id.2.0.weight (transposed)" + ], + [ + "onnx::MatMul_1652", + "nn0_id.2.4.weight (transposed)" + ], + [ + "onnx::MatMul_1653", + "nn0_id.3.0.weight (transposed)" + ], + [ + "onnx::MatMul_1654", + "nn0_id.3.4.weight (transposed)" + ], + [ + "onnx::MatMul_1655", + "nn0_id.4.0.weight (transposed)" + ], + [ + "onnx::MatMul_1656", + "nn0_id.4.4.weight (transposed)" + ], + [ + "onnx::MatMul_1657", + "nn0_id.5.0.weight (transposed)" + ], + [ + "onnx::MatMul_1658", + "nn0_id.5.4.weight (transposed)" + ], + [ + "onnx::MatMul_1659", + "nn0_id.6.0.weight (transposed)" + ], + [ + "onnx::MatMul_1660", + "nn0_id.6.4.weight (transposed)" + ], + [ + "onnx::MatMul_1661", + "nn0_id.7.0.weight (transposed)" + ], + [ + "onnx::MatMul_1662", + "nn0_id.7.4.weight (transposed)" + ], + [ + "onnx::MatMul_1667", + "nn0_reg.0.0.weight (transposed)" + ], + [ + "onnx::MatMul_1668", + "nn0_reg.0.4.weight (transposed)" + ], + [ + "onnx::MatMul_1669", + "nn0_reg.1.0.weight (transposed)" + ], + [ + "onnx::MatMul_1670", + "nn0_reg.1.4.weight (transposed)" + ], + [ + "onnx::MatMul_1671", + "nn0_reg.2.0.weight (transposed)" + ], + [ + "onnx::MatMul_1672", + "nn0_reg.2.4.weight (transposed)" + ], + [ + "onnx::MatMul_1673", + "nn0_reg.3.0.weight (transposed)" + ], + [ + "onnx::MatMul_1674", + "nn0_reg.3.4.weight (transposed)" + ], + [ + "onnx::MatMul_1675", + "nn0_reg.4.0.weight (transposed)" + ], + [ + "onnx::MatMul_1676", + "nn0_reg.4.4.weight (transposed)" + ], + [ + "onnx::MatMul_1677", + "nn0_reg.5.0.weight (transposed)" + ], + [ + "onnx::MatMul_1678", + "nn0_reg.5.4.weight (transposed)" + ], + [ + "onnx::MatMul_1679", + "nn0_reg.6.0.weight (transposed)" + ], + [ + "onnx::MatMul_1680", + "nn0_reg.6.4.weight (transposed)" + ], + [ + "onnx::MatMul_1681", + "nn0_reg.7.0.weight (transposed)" + ], + [ + "onnx::MatMul_1682", + "nn0_reg.7.4.weight (transposed)" + ], + [ + "onnx::MatMul_1695", + "conv_id.0.mha.out_proj.weight (transposed)" + ], + [ + "onnx::MatMul_1696", + "conv_id.0.seq.0.weight (transposed)" + ], + [ + "onnx::MatMul_1697", + "conv_id.0.seq.2.weight (transposed)" + ], + [ + "onnx::MatMul_1710", + "conv_id.1.mha.out_proj.weight (transposed)" + ], + [ + "onnx::MatMul_1711", + "conv_id.1.seq.0.weight (transposed)" + ], + [ + "onnx::MatMul_1712", + "conv_id.1.seq.2.weight (transposed)" + ], + [ + "onnx::MatMul_1725", + "conv_id.2.mha.out_proj.weight (transposed)" + ], + [ + "onnx::MatMul_1726", + "conv_id.2.seq.0.weight (transposed)" + ], + [ + "onnx::MatMul_1727", + "conv_id.2.seq.2.weight (transposed)" + ], + [ + "onnx::MatMul_1740", + "conv_reg.0.mha.out_proj.weight (transposed)" + ], + [ + "onnx::MatMul_1741", + "conv_reg.0.seq.0.weight (transposed)" + ], + [ + "onnx::MatMul_1742", + "conv_reg.0.seq.2.weight (transposed)" + ], + [ + "onnx::MatMul_1755", + "conv_reg.1.mha.out_proj.weight (transposed)" + ], + [ + "onnx::MatMul_1756", + "conv_reg.1.seq.0.weight (transposed)" + ], + [ + "onnx::MatMul_1757", + "conv_reg.1.seq.2.weight (transposed)" + ], + [ + "onnx::MatMul_1770", + "conv_reg.2.mha.out_proj.weight (transposed)" + ], + [ + "onnx::MatMul_1771", + "conv_reg.2.seq.0.weight (transposed)" + ], + [ + "onnx::MatMul_1772", + "conv_reg.2.seq.2.weight (transposed)" + ], + [ + "onnx::MatMul_1773", + "nn_binary_particle.0.weight (transposed)" + ], + [ + "onnx::MatMul_1774", + "nn_binary_particle.4.weight (transposed)" + ], + [ + "onnx::MatMul_1775", + "nn_pid.0.weight (transposed)" + ], + [ + "onnx::MatMul_1776", + "nn_pid.4.weight (transposed)" + ], + [ + "onnx::MatMul_1777", + "nn_pu.0.weight (transposed)" + ], + [ + "onnx::MatMul_1778", + "nn_pu.4.weight (transposed)" + ], + [ + "onnx::MatMul_1779", + "nn_pt.nn.0.0.weight (transposed)" + ], + [ + "onnx::MatMul_1780", + "nn_pt.nn.0.4.weight (transposed)" + ], + [ + "onnx::MatMul_1781", + "nn_pt.nn.1.0.weight (transposed)" + ], + [ + "onnx::MatMul_1782", + "nn_pt.nn.1.4.weight (transposed)" + ], + [ + "onnx::MatMul_1783", + "nn_pt.nn.2.0.weight (transposed)" + ], + [ + "onnx::MatMul_1784", + "nn_pt.nn.2.4.weight (transposed)" + ], + [ + "onnx::MatMul_1785", + "nn_pt.nn.3.0.weight (transposed)" + ], + [ + "onnx::MatMul_1786", + "nn_pt.nn.3.4.weight (transposed)" + ], + [ + "onnx::MatMul_1787", + "nn_pt.nn.4.0.weight (transposed)" + ], + [ + "onnx::MatMul_1788", + "nn_pt.nn.4.4.weight (transposed)" + ], + [ + "onnx::MatMul_1789", + "nn_pt.nn.5.0.weight (transposed)" + ], + [ + "onnx::MatMul_1790", + "nn_pt.nn.5.4.weight (transposed)" + ], + [ + "onnx::MatMul_1791", + "nn_pt.nn.6.0.weight (transposed)" + ], + [ + "onnx::MatMul_1792", + "nn_pt.nn.6.4.weight (transposed)" + ], + [ + "onnx::MatMul_1793", + "nn_pt.nn.7.0.weight (transposed)" + ], + [ + "onnx::MatMul_1794", + "nn_pt.nn.7.4.weight (transposed)" + ], + [ + "onnx::MatMul_1799", + "nn_eta.nn.0.weight (transposed)" + ], + [ + "onnx::MatMul_1800", + "nn_eta.nn.4.weight (transposed)" + ], + [ + "onnx::MatMul_1813", + "nn_sin_phi.nn.0.weight (transposed)" + ], + [ + "onnx::MatMul_1814", + "nn_sin_phi.nn.4.weight (transposed)" + ], + [ + "onnx::MatMul_1827", + "nn_cos_phi.nn.0.weight (transposed)" + ], + [ + "onnx::MatMul_1828", + "nn_cos_phi.nn.4.weight (transposed)" + ], + [ + "onnx::MatMul_1846", + "nn_energy.nn.0.0.weight (transposed)" + ], + [ + "onnx::MatMul_1847", + "nn_energy.nn.0.4.weight (transposed)" + ], + [ + "onnx::MatMul_1848", + "nn_energy.nn.1.0.weight (transposed)" + ], + [ + "onnx::MatMul_1849", + "nn_energy.nn.1.4.weight (transposed)" + ], + [ + "onnx::MatMul_1850", + "nn_energy.nn.2.0.weight (transposed)" + ], + [ + "onnx::MatMul_1851", + "nn_energy.nn.2.4.weight (transposed)" + ], + [ + "onnx::MatMul_1852", + "nn_energy.nn.3.0.weight (transposed)" + ], + [ + "onnx::MatMul_1853", + "nn_energy.nn.3.4.weight (transposed)" + ], + [ + "onnx::MatMul_1854", + "nn_energy.nn.4.0.weight (transposed)" + ], + [ + "onnx::MatMul_1855", + "nn_energy.nn.4.4.weight (transposed)" + ], + [ + "onnx::MatMul_1856", + "nn_energy.nn.5.0.weight (transposed)" + ], + [ + "onnx::MatMul_1857", + "nn_energy.nn.5.4.weight (transposed)" + ], + [ + "onnx::MatMul_1858", + "nn_energy.nn.6.0.weight (transposed)" + ], + [ + "onnx::MatMul_1859", + "nn_energy.nn.6.4.weight (transposed)" + ], + [ + "onnx::MatMul_1860", + "nn_energy.nn.7.0.weight (transposed)" + ], + [ + "onnx::MatMul_1861", + "nn_energy.nn.7.4.weight (transposed)" + ] + ], + "onnx_other_initializers": [], + "current_models": { + "attention_seed12345_20261027_160940_955225": { + "parameters_and_buffers": 12421170 + }, + "heptv2_seed12345_20261027_160940_124470": { + "parameters_and_buffers": 23473252 + } + } +} diff --git a/cms/v3.2.0/comparison_metrics/summary.json b/cms/v3.2.0/comparison_metrics/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0231053fccc0c6df0d9ed23eed5fdcd47d4950e3 --- /dev/null +++ b/cms/v3.2.0/comparison_metrics/summary.json @@ -0,0 +1,454 @@ +{ + "reference_onnx_sha256": "3e8316ae25aba290b9914b8167090927cb3888f15312d156575011869f748735", + "reference_url": "https://huggingface.co/jpata/particleflow/tree/main/cms/v2.6.0pre1/pyg-cms_20250722_101813_274478", + "onnxruntime": "1.29.0", + "samples": { + "cms_pf_qcd": { + "events": 512, + "models": { + "reference": { + "particle": { + "count/target_mean": 2890.13671875, + "count/prediction_mean": 3028.734375, + "count/bias_mean": 138.59765625, + "count/mae": 139.0703125, + "matching/target_coverage_dr0p05": 0.5219645210339584, + "matching/target_coverage_dr0p10": 0.7103395843892549, + "matching/target_coverage_rel_pt0p20": 0.5665781381990201, + "matching/target_coverage_rel_pt0p50": 0.7635478966041561, + "matching/efficiency": 0.6485906403108633, + "matching/purity": 0.6189105391587865, + "matching/f1": 0.6334030916738108, + "matching/duplicate_fraction": 0.10806068438239982, + "matched/pid_accuracy": 0.8261978094341038, + "matched/delta_r_mean": 0.026634588266140607, + "matched/delta_eta_abs_mean": 0.017559432938882135, + "matched/delta_phi_abs_mean": 0.016065793866967878, + "matched/pt_relative_abs_mean": 0.10391286691577047, + "matched/energy_relative_abs_mean": 0.11204425707249661, + "event/energy_response_mean": 1.0217106676820673, + "event/energy_relative_abs_error": 0.024256467714845735, + "event/scalar_pt_response_mean": 1.0494994740513897, + "event/scalar_pt_relative_abs_error": 0.04979755730560663, + "event/vector_pt_closure": 0.011628058042674368, + "event/met_abs_error": 24.87498156693988, + "class_1/efficiency": 0.9058223518788255, + "class_1/pid_accuracy": 0.9801395030110686, + "class_2/efficiency": 0.2646101379631065, + "class_2/pid_accuracy": 0.20419188960489487, + "class_3/efficiency": 0.38880933866899314, + "class_3/pid_accuracy": 0.478604441300928, + "class_4/efficiency": 0.45743879175797325, + "class_4/pid_accuracy": 0.5203578739858974, + "class_5/efficiency": 0.3739123694358855, + "class_5/pid_accuracy": 0.6205752785695776, + "class_6/efficiency": 0.8640679660169915, + "class_6/pid_accuracy": 0.010410641989589358, + "class_7/efficiency": 0.9877408056042032, + "class_7/pid_accuracy": 0.05851063829787234, + "class_8/efficiency": null, + "class_8/pid_accuracy": null + }, + "jet_response_median": 1.0279237433407529, + "jet_response_iqr": 0.38068409177597384, + "jet_angular_recall": 0.19813143933332514, + "jet_angular_precision": 0.19038381926784553 + }, + "attention": { + "particle": { + "count/target_mean": 2890.13671875, + "count/prediction_mean": 3055.904296875, + "count/bias_mean": 165.767578125, + "count/mae": 165.822265625, + "matching/target_coverage_dr0p05": 0.5151011995269471, + "matching/target_coverage_dr0p10": 0.7095137692177733, + "matching/target_coverage_rel_pt0p20": 0.5610069268457509, + "matching/target_coverage_rel_pt0p50": 0.7612603480317621, + "matching/efficiency": 0.6438824125696908, + "matching/purity": 0.6089550006615012, + "matching/f1": 0.6259318421231564, + "matching/duplicate_fraction": 0.11574353694148687, + "matched/pid_accuracy": 0.8396164927029708, + "matched/delta_r_mean": 0.028190757197628383, + "matched/delta_eta_abs_mean": 0.019094951951158495, + "matched/delta_phi_abs_mean": 0.016354801084070947, + "matched/pt_relative_abs_mean": 0.10626501046950057, + "matched/energy_relative_abs_mean": 0.11963416888549096, + "event/energy_response_mean": 1.0203297597335255, + "event/energy_relative_abs_error": 0.023157161955658503, + "event/scalar_pt_response_mean": 1.043101235365183, + "event/scalar_pt_relative_abs_error": 0.04343458501695014, + "event/vector_pt_closure": 0.012521105480240841, + "event/met_abs_error": 29.5209945251332, + "class_1/efficiency": 0.8905330430219063, + "class_1/pid_accuracy": 0.9775597281440461, + "class_2/efficiency": 0.2685630134862812, + "class_2/pid_accuracy": 0.20275773609106942, + "class_3/efficiency": 0.4008856942363495, + "class_3/pid_accuracy": 0.4768421052631579, + "class_4/efficiency": 0.46063281360848796, + "class_4/pid_accuracy": 0.6484954686088218, + "class_5/efficiency": 0.37585024126504274, + "class_5/pid_accuracy": 0.5988399071925754, + "class_6/efficiency": 0.8535732133933034, + "class_6/pid_accuracy": 0.00702576112412178, + "class_7/efficiency": 0.9754816112084063, + "class_7/pid_accuracy": 0.06104129263913824, + "class_8/efficiency": null, + "class_8/pid_accuracy": null + }, + "jet_response_median": 1.048952555174973, + "jet_response_iqr": 0.37771686552257944, + "jet_angular_recall": 0.19697435960560813, + "jet_angular_precision": 0.19026443450965472 + }, + "heptv2": { + "particle": { + "count/target_mean": 2890.13671875, + "count/prediction_mean": 3054.236328125, + "count/bias_mean": 164.099609375, + "count/mae": 164.154296875, + "matching/target_coverage_dr0p05": 0.5160412231795911, + "matching/target_coverage_dr0p10": 0.709809089373205, + "matching/target_coverage_rel_pt0p20": 0.5638891704679845, + "matching/target_coverage_rel_pt0p50": 0.7643689812468323, + "matching/efficiency": 0.6468890015205271, + "matching/purity": 0.6121326103791545, + "matching/f1": 0.6290310656841637, + "matching/duplicate_fraction": 0.1136453018316644, + "matched/pid_accuracy": 0.841034689532549, + "matched/delta_r_mean": 0.02778929276162972, + "matched/delta_eta_abs_mean": 0.01873396394296698, + "matched/delta_phi_abs_mean": 0.01621673809510576, + "matched/pt_relative_abs_mean": 0.1053005765719331, + "matched/energy_relative_abs_mean": 0.1144275168696761, + "event/energy_response_mean": 1.0223047182616625, + "event/energy_relative_abs_error": 0.024391093346068204, + "event/scalar_pt_response_mean": 1.0446105328729147, + "event/scalar_pt_relative_abs_error": 0.04500371472887303, + "event/vector_pt_closure": 0.0128951652905572, + "event/met_abs_error": 28.232848462397715, + "class_1/efficiency": 0.8977268834501932, + "class_1/pid_accuracy": 0.977754814524241, + "class_2/efficiency": 0.26756402969393206, + "class_2/pid_accuracy": 0.2037400624416621, + "class_3/efficiency": 0.40021715629563603, + "class_3/pid_accuracy": 0.47938016088331814, + "class_4/efficiency": 0.4606611909003484, + "class_4/pid_accuracy": 0.6485924120984798, + "class_5/efficiency": 0.3704242001434025, + "class_5/pid_accuracy": 0.5986659691341878, + "class_6/efficiency": 0.8555722138930535, + "class_6/pid_accuracy": 0.008177570093457943, + "class_7/efficiency": 0.9807355516637478, + "class_7/pid_accuracy": 0.05892857142857143, + "class_8/efficiency": null, + "class_8/pid_accuracy": null + }, + "jet_response_median": 1.0255487525366904, + "jet_response_iqr": 0.36232087003909597, + "jet_angular_recall": 0.18925639163455976, + "jet_angular_precision": 0.18819079792898322 + } + } + }, + "cms_pf_ttbar": { + "events": 512, + "models": { + "reference": { + "particle": { + "count/target_mean": 2897.666015625, + "count/prediction_mean": 3034.0625, + "count/bias_mean": 136.396484375, + "count/mae": 137.224609375, + "matching/target_coverage_dr0p05": 0.5225036313574031, + "matching/target_coverage_dr0p10": 0.7099888447396713, + "matching/target_coverage_rel_pt0p20": 0.5666939650378638, + "matching/target_coverage_rel_pt0p50": 0.763788879115398, + "matching/efficiency": 0.6484353989100873, + "matching/purity": 0.6192849418065712, + "matching/f1": 0.6335250218551256, + "matching/duplicate_fraction": 0.10755999588011124, + "matched/pid_accuracy": 0.8273189178625853, + "matched/delta_r_mean": 0.026501033668306556, + "matched/delta_eta_abs_mean": 0.017477886507633968, + "matched/delta_phi_abs_mean": 0.015965909872294935, + "matched/pt_relative_abs_mean": 0.10351382153802847, + "matched/energy_relative_abs_mean": 0.1117151023096923, + "event/energy_response_mean": 1.0196156429821224, + "event/energy_relative_abs_error": 0.022686429502412935, + "event/scalar_pt_response_mean": 1.0508914713726094, + "event/scalar_pt_relative_abs_error": 0.050943913416450166, + "event/vector_pt_closure": 0.011332067299257348, + "event/met_abs_error": 21.872990756798824, + "class_1/efficiency": 0.9064288251851447, + "class_1/pid_accuracy": 0.980207392757383, + "class_2/efficiency": 0.26162339930647266, + "class_2/pid_accuracy": 0.20575113166379003, + "class_3/efficiency": 0.39141257594109735, + "class_3/pid_accuracy": 0.47914230019493176, + "class_4/efficiency": 0.4542953678611652, + "class_4/pid_accuracy": 0.5218641278306702, + "class_5/efficiency": 0.3731022013029877, + "class_5/pid_accuracy": 0.6256923076923077, + "class_6/efficiency": 0.8756108396268325, + "class_6/pid_accuracy": 0.09386098427194317, + "class_7/efficiency": 0.9681668496158068, + "class_7/pid_accuracy": 0.30272108843537415, + "class_8/efficiency": null, + "class_8/pid_accuracy": null + }, + "jet_response_median": 1.0292553553056807, + "jet_response_iqr": 0.365230819372485, + "jet_angular_recall": 0.20795335992635777, + "jet_angular_precision": 0.19975241688281065 + }, + "attention": { + "particle": { + "count/target_mean": 2897.666015625, + "count/prediction_mean": 3063.169921875, + "count/bias_mean": 165.50390625, + "count/mae": 165.78125, + "matching/target_coverage_dr0p05": 0.5152281099079606, + "matching/target_coverage_dr0p10": 0.7092056173981619, + "matching/target_coverage_rel_pt0p20": 0.5610772409098109, + "matching/target_coverage_rel_pt0p50": 0.761361683197347, + "matching/efficiency": 0.6438169189238375, + "matching/purity": 0.609031315216123, + "matching/f1": 0.62594120214368, + "matching/duplicate_fraction": 0.11535933147277094, + "matched/pid_accuracy": 0.8416103939612843, + "matched/delta_r_mean": 0.028118822845865067, + "matched/delta_eta_abs_mean": 0.01908380241102101, + "matched/delta_phi_abs_mean": 0.016281239946008056, + "matched/pt_relative_abs_mean": 0.10586426758277691, + "matched/energy_relative_abs_mean": 0.11924718610018578, + "event/energy_response_mean": 1.0181292423793422, + "event/energy_relative_abs_error": 0.02146171346795333, + "event/scalar_pt_response_mean": 1.0417060372169098, + "event/scalar_pt_relative_abs_error": 0.042014162000171176, + "event/vector_pt_closure": 0.012026979158651084, + "event/met_abs_error": 23.765845605112336, + "class_1/efficiency": 0.8916104172929249, + "class_1/pid_accuracy": 0.9781268102505004, + "class_2/efficiency": 0.2669398744153155, + "class_2/pid_accuracy": 0.20171714914940475, + "class_3/efficiency": 0.40273817130976336, + "class_3/pid_accuracy": 0.4804925700076964, + "class_4/efficiency": 0.45729239291944945, + "class_4/pid_accuracy": 0.6524735603954385, + "class_5/efficiency": 0.3737527384744903, + "class_5/pid_accuracy": 0.6074280741271629, + "class_6/efficiency": 0.8680586406041759, + "class_6/pid_accuracy": 0.07574206755373593, + "class_7/efficiency": 0.960482985729967, + "class_7/pid_accuracy": 0.30057142857142854, + "class_8/efficiency": null, + "class_8/pid_accuracy": null + }, + "jet_response_median": 1.0485542740717433, + "jet_response_iqr": 0.35525660407754733, + "jet_angular_recall": 0.20708192697146363, + "jet_angular_precision": 0.1998791626683726 + }, + "heptv2": { + "particle": { + "count/target_mean": 2897.666015625, + "count/prediction_mean": 3061.1171875, + "count/bias_mean": 163.451171875, + "count/mae": 163.658203125, + "matching/target_coverage_dr0p05": 0.5164521553917654, + "matching/target_coverage_dr0p10": 0.7096140819153346, + "matching/target_coverage_rel_pt0p20": 0.5638724593136313, + "matching/target_coverage_rel_pt0p50": 0.7640861280462118, + "matching/efficiency": 0.6465723693301114, + "matching/purity": 0.6120480421006423, + "matching/f1": 0.6288366995018186, + "matching/duplicate_fraction": 0.11324947744262077, + "matched/pid_accuracy": 0.8430119946875606, + "matched/delta_r_mean": 0.027675148032157662, + "matched/delta_eta_abs_mean": 0.0186721160874133, + "matched/delta_phi_abs_mean": 0.016135954191816407, + "matched/pt_relative_abs_mean": 0.10497289670767636, + "matched/energy_relative_abs_mean": 0.11412114084514187, + "event/energy_response_mean": 1.020319929410056, + "event/energy_relative_abs_error": 0.023069513275567558, + "event/scalar_pt_response_mean": 1.0442948148434468, + "event/scalar_pt_relative_abs_error": 0.04446671599433574, + "event/vector_pt_closure": 0.012030867049616156, + "event/met_abs_error": 23.193618104669333, + "class_1/efficiency": 0.8987319979231001, + "class_1/pid_accuracy": 0.978383752998878, + "class_2/efficiency": 0.2639067572057834, + "class_2/pid_accuracy": 0.2037772397094431, + "class_3/efficiency": 0.4020276392213713, + "class_3/pid_accuracy": 0.48220767205181125, + "class_4/efficiency": 0.45674977364573904, + "class_4/pid_accuracy": 0.652530373460606, + "class_5/efficiency": 0.3695433803059438, + "class_5/pid_accuracy": 0.6021021021021021, + "class_6/efficiency": 0.8662816525988449, + "class_6/pid_accuracy": 0.08974358974358974, + "class_7/efficiency": 0.9571899012074643, + "class_7/pid_accuracy": 0.30504587155963303, + "class_8/efficiency": null, + "class_8/pid_accuracy": null + }, + "jet_response_median": 1.0266958929906298, + "jet_response_iqr": 0.3431011058542559, + "jet_angular_recall": 0.19310217858238723, + "jet_angular_precision": 0.19242435361170226 + } + } + }, + "cms_pf_ztt": { + "events": 512, + "models": { + "reference": { + "particle": { + "count/target_mean": 2804.873046875, + "count/prediction_mean": 2933.126953125, + "count/bias_mean": 128.25390625, + "count/mae": 128.98046875, + "matching/target_coverage_dr0p05": 0.5211549375215428, + "matching/target_coverage_dr0p10": 0.7082957603779694, + "matching/target_coverage_rel_pt0p20": 0.5673197107433701, + "matching/target_coverage_rel_pt0p50": 0.7642628099116006, + "matching/efficiency": 0.6468924409596858, + "matching/purity": 0.6186064227263859, + "matching/f1": 0.632433311911816, + "matching/duplicate_fraction": 0.10595227869148287, + "matched/pid_accuracy": 0.8296844237722538, + "matched/delta_r_mean": 0.026523954127018853, + "matched/delta_eta_abs_mean": 0.01746809131554697, + "matched/delta_phi_abs_mean": 0.01601598751427424, + "matched/pt_relative_abs_mean": 0.1030678088456419, + "matched/energy_relative_abs_mean": 0.11134467844696942, + "event/energy_response_mean": 1.0222973287217536, + "event/energy_relative_abs_error": 0.024755676815621734, + "event/scalar_pt_response_mean": 1.0553412870715175, + "event/scalar_pt_relative_abs_error": 0.05534128707151747, + "event/vector_pt_closure": 0.010182239332362293, + "event/met_abs_error": 16.397713788870348, + "class_1/efficiency": 0.9076171456998978, + "class_1/pid_accuracy": 0.9828865097162275, + "class_2/efficiency": 0.2526456690083907, + "class_2/pid_accuracy": 0.21679763894321374, + "class_3/efficiency": 0.39250280905591817, + "class_3/pid_accuracy": 0.4799001096357656, + "class_4/efficiency": 0.45358388515392, + "class_4/pid_accuracy": 0.5241411974479704, + "class_5/efficiency": 0.3592727033458717, + "class_5/pid_accuracy": 0.628704862488935, + "class_6/efficiency": 0.8531659388646288, + "class_6/pid_accuracy": 0.003198976327575176, + "class_7/efficiency": 0.9664179104477612, + "class_7/pid_accuracy": 0.032818532818532815, + "class_8/efficiency": null, + "class_8/pid_accuracy": null + }, + "jet_response_median": 1.0360193937900064, + "jet_response_iqr": 0.40236522848229617, + "jet_angular_recall": 0.19215487256928543, + "jet_angular_precision": 0.18457660571183981 + }, + "attention": { + "particle": { + "count/target_mean": 2804.873046875, + "count/prediction_mean": 2959.83984375, + "count/bias_mean": 154.966796875, + "count/mae": 155.091796875, + "matching/target_coverage_dr0p05": 0.5137494385817094, + "matching/target_coverage_dr0p10": 0.7074239517580662, + "matching/target_coverage_rel_pt0p20": 0.5620052990923302, + "matching/target_coverage_rel_pt0p50": 0.7624558263903154, + "matching/efficiency": 0.6421810534818378, + "matching/purity": 0.6085587137184101, + "matching/f1": 0.6249179663584992, + "matching/duplicate_fraction": 0.11310987318517815, + "matched/pid_accuracy": 0.8445414553588952, + "matched/delta_r_mean": 0.028066235349570423, + "matched/delta_eta_abs_mean": 0.018996010723135708, + "matched/delta_phi_abs_mean": 0.016304286936961485, + "matched/pt_relative_abs_mean": 0.1055525795848727, + "matched/energy_relative_abs_mean": 0.11890517189443843, + "event/energy_response_mean": 1.0186915546604012, + "event/energy_relative_abs_error": 0.021919513513209813, + "event/scalar_pt_response_mean": 1.044222482995433, + "event/scalar_pt_relative_abs_error": 0.04422248299543258, + "event/vector_pt_closure": 0.010863337915719497, + "event/met_abs_error": 17.415130965373613, + "class_1/efficiency": 0.8929860690833998, + "class_1/pid_accuracy": 0.9811332134484778, + "class_2/efficiency": 0.2555563540974181, + "class_2/pid_accuracy": 0.2102857946356382, + "class_3/efficiency": 0.40333739749934255, + "class_3/pid_accuracy": 0.4790293517947745, + "class_4/efficiency": 0.4566515442551686, + "class_4/pid_accuracy": 0.6526016479525124, + "class_5/efficiency": 0.35947009990459167, + "class_5/pid_accuracy": 0.6088959394734433, + "class_6/efficiency": 0.8422489082969432, + "class_6/pid_accuracy": 0.0019442644199611147, + "class_7/efficiency": 0.9626865671641791, + "class_7/pid_accuracy": 0.03875968992248062, + "class_8/efficiency": null, + "class_8/pid_accuracy": null + }, + "jet_response_median": 1.050952585178886, + "jet_response_iqr": 0.38454960721696474, + "jet_angular_recall": 0.18995160232209832, + "jet_angular_precision": 0.18345048534404437 + }, + "heptv2": { + "particle": { + "count/target_mean": 2804.873046875, + "count/prediction_mean": 2956.70703125, + "count/bias_mean": 151.833984375, + "count/mae": 151.927734375, + "matching/target_coverage_dr0p05": 0.5151804024107041, + "matching/target_coverage_dr0p10": 0.7078501074093287, + "matching/target_coverage_rel_pt0p20": 0.5644153067868073, + "matching/target_coverage_rel_pt0p50": 0.7649528756802301, + "matching/efficiency": 0.645354241885112, + "matching/purity": 0.6122137565941841, + "matching/f1": 0.6283473263254811, + "matching/duplicate_fraction": 0.11087080882051797, + "matched/pid_accuracy": 0.8456802511895899, + "matched/delta_r_mean": 0.027688698500096087, + "matched/delta_eta_abs_mean": 0.018662028186681077, + "matched/delta_phi_abs_mean": 0.01615865446523355, + "matched/pt_relative_abs_mean": 0.10458649808280894, + "matched/energy_relative_abs_mean": 0.11375980443259578, + "event/energy_response_mean": 1.0223444457598125, + "event/energy_relative_abs_error": 0.024510964847546658, + "event/scalar_pt_response_mean": 1.0472308700576336, + "event/scalar_pt_relative_abs_error": 0.04723200106980992, + "event/vector_pt_closure": 0.010820276829872064, + "event/met_abs_error": 17.63145317476635, + "class_1/efficiency": 0.899786408675181, + "class_1/pid_accuracy": 0.9813277844898062, + "class_2/efficiency": 0.2553856657742961, + "class_2/pid_accuracy": 0.21359223300970873, + "class_3/efficiency": 0.40337564847354707, + "class_3/pid_accuracy": 0.48058413541320943, + "class_4/efficiency": 0.4568379416670879, + "class_4/pid_accuracy": 0.6532828038339719, + "class_5/efficiency": 0.35630078848958735, + "class_5/pid_accuracy": 0.6053554939981532, + "class_6/efficiency": 0.8411572052401747, + "class_6/pid_accuracy": 0.0025957170668397143, + "class_7/efficiency": 0.9664179104477612, + "class_7/pid_accuracy": 0.04054054054054054, + "class_8/efficiency": null, + "class_8/pid_accuracy": null + }, + "jet_response_median": 1.0255522306353952, + "jet_response_iqr": 0.3843649145070648, + "jet_angular_recall": 0.17912091992721782, + "jet_angular_precision": 0.17831969637223974 + } + } + } + } +} diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/checkpoints/checkpoint-100000.pth b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/checkpoints/checkpoint-100000.pth new file mode 100644 index 0000000000000000000000000000000000000000..27ff3295c344b7ea12eb34f5eb45fe7e5ac400c4 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/checkpoints/checkpoint-100000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ffec8bfb5c8b409282c32bcfd1f4bd53b47de21ad500e878273150436af9da8 +size 281697859 diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_10000.json b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..bfe9aa1bc31d1112a936363514db875abb984bfc --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 0.043092675107085296, "train": {"Classification_binary": 3.7678017616271973, "Classification": 0.01108646485954523, "Regression_pt": 0.2744787037372589, "Regression_eta": 0.07357916980981827, "Regression_sin_phi": 0.052070245146751404, "Regression_cos_phi": 0.05197954922914505, "Regression_energy": 0.3372364342212677, "Total": 13.05141830444336, "Total_unweighted": 4.568232328630984}, "train_group": {"physical_pu": {"Classification_binary": 3.0703021772615204, "Classification": 0.015771619719709015, "Regression_pt": 0.24383421749859066, "Regression_eta": 0.1073251411646277, "Regression_sin_phi": 0.04776083399633785, "Regression_cos_phi": 0.04783506011897391, "Regression_energy": 0.2908515465128553, "Total": 12.218533295851488, "Total_unweighted": 3.823680596272615}, "physical_nopu": {"Classification_binary": 3.7136507585103757, "Classification": 0.015055586175114015, "Regression_pt": 0.41543548676309283, "Regression_eta": 0.07199585112785087, "Regression_sin_phi": 0.0497371791006446, "Regression_cos_phi": 0.04974170427511234, "Regression_energy": 0.46403496762444607, "Total": 14.607822066748758, "Total_unweighted": 4.7796515335766365}}, "valid": {"Classification_binary": 3.380192438761393, "Classification": 0.01545055458943049, "Regression_pt": 0.33102450768152875, "Regression_eta": 0.09015671412150066, "Regression_sin_phi": 0.04897112647692362, "Regression_cos_phi": 0.0487969716389974, "Regression_energy": 0.3777894973754883, "Total": 13.431194305419922, "group/physical_pu/Classification": 0.015807971358299255, "group/physical_pu/Classification_binary": 3.0574893951416016, "group/physical_pu/Regression_cos_phi": 0.047959079345067344, "group/physical_pu/Regression_energy": 0.2907902201016744, "group/physical_pu/Regression_eta": 0.10707638661066692, "group/physical_pu/Regression_pt": 0.24444629748662314, "group/physical_pu/Regression_sin_phi": 0.04789319137732188, "group/physical_pu/Total": 12.217055002848307, "group/physical_pu/Total_unweighted": 3.8114625414212546, "group/physical_nopu/Classification": 0.01509313533703486, "group/physical_nopu/Classification_binary": 3.7028942108154297, "group/physical_nopu/Regression_cos_phi": 0.04963487386703491, "group/physical_nopu/Regression_energy": 0.4647887547810872, "group/physical_nopu/Regression_eta": 0.07323705653349559, "group/physical_nopu/Regression_pt": 0.4176026980082194, "group/physical_nopu/Regression_sin_phi": 0.05004907647768656, "group/physical_nopu/Total": 14.645334879557291, "group/physical_nopu/Total_unweighted": 4.773299805819988, "Total_unweighted": 4.292381810645263, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.015260087326169014, "diagnostic/loss/cms_pf/Classification_binary": 3.3751935958862305, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.04837040603160858, "diagnostic/loss/cms_pf/Regression_energy": 0.3645273745059967, "diagnostic/loss/cms_pf/Regression_eta": 0.09022770822048187, "diagnostic/loss/cms_pf/Regression_pt": 0.3177054822444916, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.04718884825706482, "diagnostic/regression/cms_pf/energy_pred_mean": 0.20963068306446075, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.3654497563838959, "diagnostic/regression/cms_pf/energy_residual_mean": 0.09886446595191956, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5994340777397156, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.11921705305576324, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.3392012417316437, "diagnostic/regression/cms_pf/pt_residual_mean": 0.12998923659324646, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5604549050331116, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8827412724494934, "metrics/particle/class_1/pid_accuracy": 0.9792342185974121, "metrics/particle/class_2/efficiency": 0.2525866627693176, "metrics/particle/class_2/pid_accuracy": 0.1971871256828308, "metrics/particle/class_3/efficiency": 0.38698136806488037, "metrics/particle/class_3/pid_accuracy": 0.4591503441333771, "metrics/particle/class_4/efficiency": 0.42368292808532715, "metrics/particle/class_4/pid_accuracy": 0.6476302146911621, "metrics/particle/class_5/efficiency": 0.3541506826877594, "metrics/particle/class_5/pid_accuracy": 0.6033750772476196, "metrics/particle/class_6/efficiency": 0.8598554730415344, "metrics/particle/class_6/pid_accuracy": 0.047132499516010284, "metrics/particle/class_7/efficiency": 0.9658458828926086, "metrics/particle/class_7/pid_accuracy": 0.2397203892469406, "metrics/particle/count/bias_mean": 62.64973831176758, "metrics/particle/count/mae": 86.05403900146484, "metrics/particle/count/prediction_mean": 1591.2532958984375, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.09077136963605881, "metrics/particle/event/energy_response_mean": 0.9243981242179871, "metrics/particle/event/met_abs_error": 26.494077682495117, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05126190185546875, "metrics/particle/event/scalar_pt_response_mean": 1.0060385465621948, "metrics/particle/event/vector_pt_closure": 0.034720078110694885, "metrics/particle/matched/delta_eta_abs_mean": 0.0204695463180542, "metrics/particle/matched/delta_phi_abs_mean": 0.018634846433997154, "metrics/particle/matched/delta_r_mean": 0.030784742906689644, "metrics/particle/matched/energy_relative_abs_mean": 0.11344540864229202, "metrics/particle/matched/pid_accuracy": 0.8436776995658875, "metrics/particle/matched/pt_relative_abs_mean": 0.1060217097401619, "metrics/particle/matching/duplicate_fraction": 0.11001978069543839, "metrics/particle/matching/efficiency": 0.6268659234046936, "metrics/particle/matching/f1": 0.6142778992652893, "metrics/particle/matching/purity": 0.6021854281425476, "metrics/particle/matching/target_coverage_dr0p05": 0.5083364844322205, "metrics/particle/matching/target_coverage_dr0p10": 0.6943060755729675, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5485138893127441, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7446715831756592, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020827986299991608, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06397455185651779, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05699797719717026, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0019272257341071963, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1059534102678299, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0018352153711020947, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.21035723388195038, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.013334334827959538, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2498658895492554, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 67567.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.49629396200180054, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2788429260253906, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.41519150137901306, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.29068660736083984, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1471034586429596, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.2244713306427, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.04430675879120827, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.0109572410583496, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.1181604415178299, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.938393592834473, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87639.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7708640098571777, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 57.12074661254883, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 45.72225570678711, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 19.668485641479492, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 2.617791175842285, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.5132312774658203, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 37.93825912475586, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.265507698059082, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 57.869293212890625, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04279116168618202, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16207073628902435, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16056524217128754, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.017828160896897316, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2140974998474121, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17478273808956146, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.464729309082031, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.06081875041127205, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.399351119995117, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 118633.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5788171291351318, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.010594129562378, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9988466501235962, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7769935131072998, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.33235272765159607, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 4.421913146972656, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.38832563161849976, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 16.538084030151367, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.24854137003421783, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 34.117408752441406, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228432.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5128722786903381, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -223.4140625, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 466.66473388671875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 23.377046585083008, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 7.097641944885254, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 3.511232376098633, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 464.3867492675781, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 233.466064453125, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 690.4529418945312, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.017955508083105087, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.07034220546483994, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0512174516916275, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013280732557177544, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10615916550159454, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004736770933959633, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.07533245533704758, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.008869857527315617, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6066228151321411, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59112.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3465564548969269, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3591185808181763, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.27126577496528625, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.178788959980011, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11287172883749008, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 2.1512045860290527, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.025444271042943, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.1703938245773315, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.08200180530548096, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 7.077954292297363, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72197.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.818748414516449, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.054753780364990234, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 29.20119285583496, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 21.326696395874023, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 5.969727516174316, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -3.867619514465332, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 16.902355194091797, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -3.729344606399536, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 48.637420654296875, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.030757218599319458, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11294832080602646, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10569024085998535, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.005952315405011177, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16725680232048035, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08514463156461716, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.7188377380371094, "metrics/particle/momentum/all/accepted/pz_bias": 0.03522203117609024, "metrics/particle/momentum/all/accepted/pz_residual_rms": 7.959931373596191, "metrics/particle/momentum/all/accepted/samples": 245312.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.1053379774093628, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1359493732452393, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7318143248558044, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5559914708137512, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.24972699582576752, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.6716930866241455, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.2431979924440384, "metrics/particle/momentum/all/assigned/pz_abs_mean": 10.401487350463867, "metrics/particle/momentum/all/assigned/pz_bias": 0.10430698096752167, "metrics/particle/momentum/all/assigned/pz_residual_rms": 26.766239166259766, "metrics/particle/momentum/all/assigned/samples": 388268.0, "metrics/particle/momentum/all/efficiency": 0.6271398067474365, "metrics/particle/momentum/all/event/energy_bias": -166.23855590820312, "metrics/particle/momentum/all/event/p3_error_mean": 469.232177734375, "metrics/particle/momentum/all/event/pt_vector_error_mean": 36.29021072387695, "metrics/particle/momentum/all/event/px_bias": 15.685162544250488, "metrics/particle/momentum/all/event/py_bias": 1.156843900680542, "metrics/particle/momentum/all/event/pz_abs_mean": 463.2066650390625, "metrics/particle/momentum/all/event/pz_bias": 231.00221252441406, "metrics/particle/momentum/all/event/pz_residual_rms": 690.8945922851562, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01842695102095604, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06085089594125748, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.0521000474691391, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00740852439776063, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09418486058712006, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011394070461392403, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.20061561465263367, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.004663585219532251, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.975495457649231, "metrics/particle/momentum/class_1/accepted/samples": 170264.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.24439774453639984, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.96474689245224, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3404829204082489, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2827989161014557, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0831587016582489, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.145648241043091, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.02537502348423004, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.273393154144287, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.008093660697340965, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 8.225629806518555, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.883167028427124, "metrics/particle/momentum/class_1/event/energy_bias": 28.539098739624023, "metrics/particle/momentum/class_1/event/p3_error_mean": 60.429542541503906, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 19.931987762451172, "metrics/particle/momentum/class_1/event/px_bias": 1.1479336023330688, "metrics/particle/momentum/class_1/event/py_bias": 1.1475216150283813, "metrics/particle/momentum/class_1/event/pz_abs_mean": 53.225341796875, "metrics/particle/momentum/class_1/event/pz_bias": -3.0685319900512695, "metrics/particle/momentum/class_1/event/pz_residual_rms": 85.34577178955078, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.05873527750372887, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2398732453584671, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2322288304567337, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06907560676336288, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27239635586738586, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.10543657094240189, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 3.996927261352539, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.1310236006975174, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 8.978745460510254, "metrics/particle/momentum/class_2/accepted/samples": 7891.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.2114784717559814, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.61005699634552, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1630312204360962, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.689005434513092, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4757118821144104, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.8168342113494873, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.3592642843723297, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 17.12563133239746, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.028526321053504944, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 35.85343551635742, "metrics/particle/momentum/class_2/assigned/samples": 31044.0, "metrics/particle/momentum/class_2/efficiency": 0.25400760769844055, "metrics/particle/momentum/class_2/event/energy_bias": -150.0946044921875, "metrics/particle/momentum/class_2/event/p3_error_mean": 183.8606719970703, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 31.024620056152344, "metrics/particle/momentum/class_2/event/px_bias": 5.951115131378174, "metrics/particle/momentum/class_2/event/py_bias": -4.2679877281188965, "metrics/particle/momentum/class_2/event/pz_abs_mean": 176.55154418945312, "metrics/particle/momentum/class_2/event/pz_bias": -86.67023468017578, "metrics/particle/momentum/class_2/event/pz_residual_rms": 255.58697509765625, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.06071602925658226, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.2294386625289917, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.2266026884317398, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.013276630081236362, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2659766674041748, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.323940247297287, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.577874183654785, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.18418270349502563, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 14.19651985168457, "metrics/particle/momentum/class_3/accepted/samples": 20689.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.083721160888672, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.9755141735076904, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.1395514011383057, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.9558407664299011, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.43461766839027405, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 3.226315498352051, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5489485859870911, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.94406509399414, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.12139224261045456, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 34.48834228515625, "metrics/particle/momentum/class_3/assigned/samples": 52453.0, "metrics/particle/momentum/class_3/efficiency": 0.3881540596485138, "metrics/particle/momentum/class_3/event/energy_bias": 833.250732421875, "metrics/particle/momentum/class_3/event/p3_error_mean": 296.6745300292969, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.140331268310547, "metrics/particle/momentum/class_3/event/px_bias": 0.007054727524518967, "metrics/particle/momentum/class_3/event/py_bias": 1.6560769081115723, "metrics/particle/momentum/class_3/event/pz_abs_mean": 296.27825927734375, "metrics/particle/momentum/class_3/event/pz_bias": 48.28672790527344, "metrics/particle/momentum/class_3/event/pz_residual_rms": 467.3646240234375, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.06014003977179527, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.2342272698879242, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.2319759726524353, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.06407006084918976, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2708318531513214, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.3633473217487335, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 10.952141761779785, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.07044769823551178, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.04007911682129, "metrics/particle/momentum/class_4/accepted/samples": 35539.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.9546047449111938, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0751010179519653, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.1120500564575195, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8214956521987915, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3851969838142395, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.3402106761932373, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5462700724601746, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 23.884693145751953, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.49425625801086426, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 43.15814971923828, "metrics/particle/momentum/class_4/assigned/samples": 82267.0, "metrics/particle/momentum/class_4/efficiency": 0.4216577410697937, "metrics/particle/momentum/class_4/event/energy_bias": -897.4401245117188, "metrics/particle/momentum/class_4/event/p3_error_mean": 453.69622802734375, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 12.473204612731934, "metrics/particle/momentum/class_4/event/px_bias": 2.3393607139587402, "metrics/particle/momentum/class_4/event/py_bias": 3.229743003845215, "metrics/particle/momentum/class_4/event/pz_abs_mean": 453.1819152832031, "metrics/particle/momentum/class_4/event/pz_bias": 282.6800842285156, "metrics/particle/momentum/class_4/event/pz_residual_rms": 661.9163208007812, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.05249016731977463, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22748766839504242, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2185029536485672, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03875342011451721, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2587954103946686, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.023153575137257576, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.3009629249572754, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.02613036334514618, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.8817243576049805, "metrics/particle/momentum/class_5/accepted/samples": 10236.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4893749952316284, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.241849899291992, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 1.064069151878357, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.7602649927139282, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.4011816680431366, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 6.969156265258789, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.15956050157546997, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.787862300872803, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.31160733103752136, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 18.15877342224121, "metrics/particle/momentum/class_5/assigned/samples": 28942.0, "metrics/particle/momentum/class_5/efficiency": 0.3536117672920227, "metrics/particle/momentum/class_5/event/energy_bias": 30.827577590942383, "metrics/particle/momentum/class_5/event/p3_error_mean": 116.13076782226562, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 26.223163604736328, "metrics/particle/momentum/class_5/event/px_bias": 6.396144866943359, "metrics/particle/momentum/class_5/event/py_bias": 0.5195890069007874, "metrics/particle/momentum/class_5/event/pz_abs_mean": 107.97005462646484, "metrics/particle/momentum/class_5/event/pz_bias": -11.06740951538086, "metrics/particle/momentum/class_5/event/pz_residual_rms": 165.25082397460938, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.02013218216598034, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.09510228037834167, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.09657011181116104, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.04976234585046768, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.13956817984580994, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.01807228848338127, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.9968532919883728, "metrics/particle/momentum/class_6/accepted/pz_bias": 0.4576258361339569, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 4.884260654449463, "metrics/particle/momentum/class_6/accepted/samples": 498.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.3223922550678253, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 0.9797696471214294, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.5739533305168152, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.411498486995697, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.10608695447444916, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 2.9256536960601807, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.05043478310108185, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 2.4093291759490967, "metrics/particle/momentum/class_6/assigned/pz_bias": 0.05479728430509567, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 9.407093048095703, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8660869598388672, "metrics/particle/momentum/class_6/event/energy_bias": -9.96841049194336, "metrics/particle/momentum/class_6/event/p3_error_mean": 8.272839546203613, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 3.194983959197998, "metrics/particle/momentum/class_6/event/px_bias": -0.21793146431446075, "metrics/particle/momentum/class_6/event/py_bias": -1.1152721643447876, "metrics/particle/momentum/class_6/event/pz_abs_mean": 7.189145088195801, "metrics/particle/momentum/class_6/event/pz_bias": 0.4332587420940399, "metrics/particle/momentum/class_6/event/pz_residual_rms": 19.772960662841797, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.017444439232349396, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.036073118448257446, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.034555986523628235, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.016739198938012123, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.07241261005401611, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.3499985933303833, "metrics/particle/momentum/class_7/accepted/pz_bias": 0.014739681966602802, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 0.8970648050308228, "metrics/particle/momentum/class_7/accepted/samples": 195.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.04777662828564644, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.0886349231004715, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.04870151728391647, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.02552836574614048, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.015075377188622952, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.13458524644374847, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.4064103364944458, "metrics/particle/momentum/class_7/assigned/pz_bias": 0.059210438281297684, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 1.0456043481826782, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.979899525642395, "metrics/particle/momentum/class_7/event/energy_bias": -1.3528696298599243, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.117527723312378, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 0.905452311038971, "metrics/particle/momentum/class_7/event/px_bias": 0.061484403908252716, "metrics/particle/momentum/class_7/event/py_bias": -0.012826632708311081, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.7430360317230225, "metrics/particle/momentum/class_7/event/pz_bias": 0.40833598375320435, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.625544548034668, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 3.8989102840423584, "step_valid_time": 2325.7830646038055, "step_total_time": 2329.681975364685, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.050140500946397, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.39076051769548825, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 85083, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16378, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13015, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.20101871739797483, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19249438783305714, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19666422507474873, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8075056121669428, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15974225222460878, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15296827803438995, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15628189579605903, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.20101871739797483, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.041430086846061, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.40339882114994674, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84806, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15438, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12153, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19003188123930626, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.1820390066740561, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18594959197807823, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.817960993325944, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14959563756323932, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14330353984387897, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14638200487819567, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19003188123930626, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0468420849978384, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.41438714987253333, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84265, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 14914, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11751, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.1846043396997116, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.1769892600723907, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18071661395664448, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8230107399276093, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14545297008256075, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.13945291639470717, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.1423897633501763, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.1846043396997116} \ No newline at end of file diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_100000.json b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_100000.json new file mode 100644 index 0000000000000000000000000000000000000000..89f5885af925c0242c8417f1fc81a843144d1b76 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_100000.json @@ -0,0 +1 @@ +{"epoch": 0.430926751070853, "train": {"Classification_binary": 2.905735969543457, "Classification": 0.015849346294999123, "Regression_pt": 0.24020911753177643, "Regression_eta": 0.10380372405052185, "Regression_sin_phi": 0.048379458487033844, "Regression_cos_phi": 0.04747385159134865, "Regression_energy": 0.2835644483566284, "Total": 11.948795318603516, "Total_unweighted": 3.6450159158557653}, "train_group": {"physical_pu": {"Classification_binary": 2.9041684014456615, "Classification": 0.015507478256720109, "Regression_pt": 0.22727172649823701, "Regression_eta": 0.10487501798095283, "Regression_sin_phi": 0.04720890014381199, "Regression_cos_phi": 0.0472074830188201, "Regression_energy": 0.2738514352630783, "Total": 11.751239965250203, "Total_unweighted": 3.6200904426072817}, "physical_nopu": {"Classification_binary": 3.4536728882553556, "Classification": 0.0141772861501286, "Regression_pt": 0.36412531534175013, "Regression_eta": 0.06919589945257562, "Regression_sin_phi": 0.04769924340402857, "Regression_cos_phi": 0.04770785209910311, "Regression_energy": 0.4104618089820686, "Total": 13.464891023499476, "Total_unweighted": 4.40704029368501}}, "valid": {"Classification_binary": 3.1787182490030923, "Classification": 0.014830179512500763, "Regression_pt": 0.2949534257253011, "Regression_eta": 0.0875298281510671, "Regression_sin_phi": 0.04767018059889475, "Regression_cos_phi": 0.04753386974334717, "Regression_energy": 0.3408623933792114, "Total": 12.614317576090494, "group/physical_pu/Classification": 0.015458807349205017, "group/physical_pu/Classification_binary": 2.901826858520508, "group/physical_pu/Regression_cos_phi": 0.04727804660797119, "group/physical_pu/Regression_energy": 0.2732653021812439, "group/physical_pu/Regression_eta": 0.1044339934984843, "group/physical_pu/Regression_pt": 0.2266618808110555, "group/physical_pu/Regression_sin_phi": 0.04722161591053009, "group/physical_pu/Total": 11.736282348632812, "group/physical_pu/Total_unweighted": 3.616146504878998, "group/physical_nopu/Classification": 0.014201554159323374, "group/physical_nopu/Classification_binary": 3.4556102752685547, "group/physical_nopu/Regression_cos_phi": 0.04778969784577688, "group/physical_nopu/Regression_energy": 0.40845954418182373, "group/physical_nopu/Regression_eta": 0.07062565286954244, "group/physical_nopu/Regression_pt": 0.3632449309031169, "group/physical_nopu/Regression_sin_phi": 0.04811875522136688, "group/physical_nopu/Total": 13.492352803548178, "group/physical_nopu/Total_unweighted": 4.408050410449505, "Total_unweighted": 4.012098126113415, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.014652466401457787, "diagnostic/loss/cms_pf/Classification_binary": 3.1774539947509766, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.04715732857584953, "diagnostic/loss/cms_pf/Regression_energy": 0.3278619945049286, "diagnostic/loss/cms_pf/Regression_eta": 0.08764075487852097, "diagnostic/loss/cms_pf/Regression_pt": 0.28201740980148315, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.045963577926158905, "diagnostic/regression/cms_pf/energy_pred_mean": 0.19988587498664856, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.3510324954986572, "diagnostic/regression/cms_pf/energy_residual_mean": 0.08911964297294617, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5820804834365845, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.11303076148033142, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.3238222599029541, "diagnostic/regression/cms_pf/pt_residual_mean": 0.12380293756723404, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5419010519981384, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8997453451156616, "metrics/particle/class_1/pid_accuracy": 0.9787273406982422, "metrics/particle/class_2/efficiency": 0.25664815306663513, "metrics/particle/class_2/pid_accuracy": 0.21084167063236237, "metrics/particle/class_3/efficiency": 0.3954804241657257, "metrics/particle/class_3/pid_accuracy": 0.47908738255500793, "metrics/particle/class_4/efficiency": 0.4412412643432617, "metrics/particle/class_4/pid_accuracy": 0.6602423191070557, "metrics/particle/class_5/efficiency": 0.37654393911361694, "metrics/particle/class_5/pid_accuracy": 0.6359609961509705, "metrics/particle/class_6/efficiency": 0.8656653165817261, "metrics/particle/class_6/pid_accuracy": 0.06302177160978317, "metrics/particle/class_7/efficiency": 0.9674344658851624, "metrics/particle/class_7/pid_accuracy": 0.25164204835891724, "metrics/particle/count/bias_mean": 69.12239837646484, "metrics/particle/count/mae": 90.82357025146484, "metrics/particle/count/prediction_mean": 1597.7259521484375, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.08310552686452866, "metrics/particle/event/energy_response_mean": 0.9419429898262024, "metrics/particle/event/met_abs_error": 18.393125534057617, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04708262160420418, "metrics/particle/event/scalar_pt_response_mean": 1.0082036256790161, "metrics/particle/event/vector_pt_closure": 0.027102457359433174, "metrics/particle/matched/delta_eta_abs_mean": 0.018316054716706276, "metrics/particle/matched/delta_phi_abs_mean": 0.015861209481954575, "metrics/particle/matched/delta_r_mean": 0.02717730775475502, "metrics/particle/matched/energy_relative_abs_mean": 0.11311549693346024, "metrics/particle/matched/pid_accuracy": 0.8477141261100769, "metrics/particle/matched/pt_relative_abs_mean": 0.10419470816850662, "metrics/particle/matching/duplicate_fraction": 0.10930860042572021, "metrics/particle/matching/efficiency": 0.6421627998352051, "metrics/particle/matching/f1": 0.6279647350311279, "metrics/particle/matching/purity": 0.6143808960914612, "metrics/particle/matching/target_coverage_dr0p05": 0.5173667669296265, "metrics/particle/matching/target_coverage_dr0p10": 0.7040228843688965, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.561550498008728, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.759401798248291, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01677176170051098, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06699693948030472, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05817333981394768, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0022647338919341564, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.11066406220197678, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.001948980032466352, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.2194078415632248, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.012164796702563763, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.3245837688446045, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 69267.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4606754183769226, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1245782375335693, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.3661068379878998, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.24295665323734283, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12878234684467316, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.5985262393951416, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03964903578162193, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8125578165054321, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.108502596616745, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.236164093017578, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87644.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7902591228485107, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 35.31589889526367, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 41.92267990112305, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 16.30547332763672, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.1377972960472107, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.55388605594635, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 36.60246276855469, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -3.9207024574279785, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 55.33781814575195, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03992118313908577, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15958838164806366, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15733544528484344, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.012763112783432007, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2110193818807602, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17307408154010773, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.464531898498535, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.009095236659049988, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.580279350280762, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 121890.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5157514810562134, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9474545121192932, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9359518885612488, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7231690287590027, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.31763723492622375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3.5014638900756836, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.376872718334198, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.64367961883545, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.06833203136920929, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 31.892820358276367, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228613.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.526952862739563, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 229.26983642578125, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 371.1571044921875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 19.369516372680664, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.03648844361305237, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -1.5569748878479004, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 369.1549072265625, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -43.291404724121094, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 555.5333251953125, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012891016900539398, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.07184150815010071, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.04912970960140228, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010571761056780815, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10688921809196472, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008667967049404979, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.07839163392782211, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005693069659173489, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7931450605392456, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59991.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3317846357822418, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.2558391094207764, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.2535606026649475, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.16016818583011627, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10197099298238754, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 2.0582330226898193, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.022923389449715614, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.079464316368103, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0766565129160881, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.635693073272705, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72197.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.830923318862915, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -5.962010383605957, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 20.352479934692383, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 15.336874008178711, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 2.7214279174804688, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.915332317352295, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 11.62260627746582, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.1328520774841309, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 22.61406707763672, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02707991935312748, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11309168487787247, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10413958132266998, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.0030444753356277943, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1664876788854599, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08474285900592804, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.731346845626831, "metrics/particle/momentum/all/accepted/pz_bias": 0.006409385707229376, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.106694221496582, "metrics/particle/momentum/all/accepted/samples": 251148.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.057653546333313, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0447331666946411, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6805546283721924, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5101845264434814, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2349441647529602, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.0864598751068115, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.23500336706638336, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.816201210021973, "metrics/particle/momentum/all/assigned/pz_bias": -0.07894258201122284, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.020896911621094, "metrics/particle/momentum/all/assigned/samples": 388454.0, "metrics/particle/momentum/all/efficiency": 0.6420595049858093, "metrics/particle/momentum/all/event/energy_bias": 258.62371826171875, "metrics/particle/momentum/all/event/p3_error_mean": 372.73712158203125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 26.361391067504883, "metrics/particle/momentum/all/event/px_bias": 2.620119094848633, "metrics/particle/momentum/all/event/py_bias": -1.9184212684631348, "metrics/particle/momentum/all/event/pz_abs_mean": 368.7738037109375, "metrics/particle/momentum/all/event/pz_bias": -48.3449592590332, "metrics/particle/momentum/all/event/pz_residual_rms": 555.5001220703125, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.013171413913369179, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06172305345535278, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.050977062433958054, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009306180290877819, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09580947458744049, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001244339975528419, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.20005933940410614, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.002151034539565444, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1514254808425903, "metrics/particle/momentum/class_1/accepted/samples": 173586.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.21638843417167664, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.8298962712287903, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2937922477722168, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.23851560056209564, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06839637458324432, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.4922266006469727, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.022522149607539177, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.108363151550293, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.008516049943864346, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.409151077270508, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.9003983736038208, "metrics/particle/momentum/class_1/event/energy_bias": 33.36625671386719, "metrics/particle/momentum/class_1/event/p3_error_mean": 50.678062438964844, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.1831693649292, "metrics/particle/momentum/class_1/event/px_bias": 0.16369032859802246, "metrics/particle/momentum/class_1/event/py_bias": -1.2255812883377075, "metrics/particle/momentum/class_1/event/pz_abs_mean": 45.84054183959961, "metrics/particle/momentum/class_1/event/pz_bias": -1.1521189212799072, "metrics/particle/momentum/class_1/event/pz_residual_rms": 74.16901397705078, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.05797980725765228, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.24149338901042938, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23307296633720398, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07233625650405884, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27304089069366455, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.11344113200902939, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.2597784996032715, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.11375255882740021, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 10.561776161193848, "metrics/particle/momentum/class_2/accepted/samples": 7916.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.189150810241699, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.5361381769180298, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1126861572265625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.625358521938324, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.47138577699661255, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.658123254776001, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.35634922981262207, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.5958309173584, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.2002575546503067, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 34.37158203125, "metrics/particle/momentum/class_2/assigned/samples": 31051.0, "metrics/particle/momentum/class_2/efficiency": 0.2548123300075531, "metrics/particle/momentum/class_2/event/energy_bias": -285.71441650390625, "metrics/particle/momentum/class_2/event/p3_error_mean": 165.034912109375, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 28.65186309814453, "metrics/particle/momentum/class_2/event/px_bias": 2.094637870788574, "metrics/particle/momentum/class_2/event/py_bias": -2.6435976028442383, "metrics/particle/momentum/class_2/event/pz_abs_mean": 158.833251953125, "metrics/particle/momentum/class_2/event/pz_bias": -33.00663757324219, "metrics/particle/momentum/class_2/event/pz_residual_rms": 223.6172332763672, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.060715895146131516, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.22617395222187042, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.22293493151664734, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.006742442492395639, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.26234808564186096, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.3150619864463806, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.385721206665039, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.044916097074747086, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 14.093539237976074, "metrics/particle/momentum/class_3/accepted/samples": 21053.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.0369019508361816, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.9547109603881836, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.090943455696106, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.915334939956665, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4220143258571625, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 2.8094582557678223, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5350773334503174, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.405500411987305, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.04073617607355118, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.55659103393555, "metrics/particle/momentum/class_3/assigned/samples": 52484.0, "metrics/particle/momentum/class_3/efficiency": 0.3949832022190094, "metrics/particle/momentum/class_3/event/energy_bias": 614.9312744140625, "metrics/particle/momentum/class_3/event/p3_error_mean": 285.8134460449219, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.541104316711426, "metrics/particle/momentum/class_3/event/px_bias": 0.22583015263080597, "metrics/particle/momentum/class_3/event/py_bias": -0.41989561915397644, "metrics/particle/momentum/class_3/event/pz_abs_mean": 285.312255859375, "metrics/particle/momentum/class_3/event/pz_bias": 31.541385650634766, "metrics/particle/momentum/class_3/event/pz_residual_rms": 453.4063720703125, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.05973006412386894, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.23031948506832123, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22717462480068207, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.05546547845005989, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.26631104946136475, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.35911110043525696, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 10.969493865966797, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.04428321123123169, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.178300857543945, "metrics/particle/momentum/class_4/accepted/samples": 36944.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.8506394624710083, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0454347133636475, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0621451139450073, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.7909663319587708, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.36744844913482666, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.320012331008911, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5273062586784363, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.151023864746094, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.19613659381866455, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 39.400753021240234, "metrics/particle/momentum/class_4/assigned/samples": 82417.0, "metrics/particle/momentum/class_4/efficiency": 0.4383275508880615, "metrics/particle/momentum/class_4/event/energy_bias": -69.73394775390625, "metrics/particle/momentum/class_4/event/p3_error_mean": 322.90911865234375, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 10.906964302062988, "metrics/particle/momentum/class_4/event/px_bias": -0.7768374681472778, "metrics/particle/momentum/class_4/event/py_bias": -0.1492684781551361, "metrics/particle/momentum/class_4/event/pz_abs_mean": 322.3055114746094, "metrics/particle/momentum/class_4/event/pz_bias": -39.82192611694336, "metrics/particle/momentum/class_4/event/pz_residual_rms": 476.96954345703125, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.051085423678159714, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22410202026367188, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21208423376083374, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.036244478076696396, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2526400089263916, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02372046373784542, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2995277643203735, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.04136589542031288, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.254275798797607, "metrics/particle/momentum/class_5/accepted/samples": 10961.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4343020915985107, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.1042590141296387, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9651241898536682, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6633520722389221, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.37771251797676086, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 5.106754302978516, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.148686945438385, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.422821044921875, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.28672993183135986, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 17.438405990600586, "metrics/particle/momentum/class_5/assigned/samples": 28940.0, "metrics/particle/momentum/class_5/efficiency": 0.3786575496196747, "metrics/particle/momentum/class_5/event/energy_bias": -24.40421485900879, "metrics/particle/momentum/class_5/event/p3_error_mean": 111.95899963378906, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 26.04820442199707, "metrics/particle/momentum/class_5/event/px_bias": 1.5952370166778564, "metrics/particle/momentum/class_5/event/py_bias": 2.941323757171631, "metrics/particle/momentum/class_5/event/pz_abs_mean": 103.8230209350586, "metrics/particle/momentum/class_5/event/pz_bias": -4.546812057495117, "metrics/particle/momentum/class_5/event/pz_residual_rms": 162.0972137451172, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.017658047378063202, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.08526857197284698, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.08519330620765686, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.03386756405234337, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.12662586569786072, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.014198782853782177, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.6551172733306885, "metrics/particle/momentum/class_6/accepted/pz_bias": -0.021125933155417442, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 3.0862174034118652, "metrics/particle/momentum/class_6/accepted/samples": 493.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.3697136640548706, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 1.661255121231079, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.7584425210952759, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.6074490547180176, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.12869565188884735, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 4.193052291870117, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.059130433946847916, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 2.736356019973755, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.7460542321205139, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 11.50854778289795, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8573912978172302, "metrics/particle/momentum/class_6/event/energy_bias": -8.524418830871582, "metrics/particle/momentum/class_6/event/p3_error_mean": 7.163695335388184, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 2.672304630279541, "metrics/particle/momentum/class_6/event/px_bias": -0.5608617067337036, "metrics/particle/momentum/class_6/event/py_bias": -0.3329479694366455, "metrics/particle/momentum/class_6/event/pz_abs_mean": 6.300057411193848, "metrics/particle/momentum/class_6/event/pz_bias": -1.1000888347625732, "metrics/particle/momentum/class_6/event/pz_residual_rms": 17.762420654296875, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.011045962572097778, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.04024937003850937, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.03946591541171074, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.015710417181253433, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.07650063186883926, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.010256410576403141, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.526902437210083, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.26024454832077026, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 2.1845595836639404, "metrics/particle/momentum/class_7/accepted/samples": 195.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.03589008376002312, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.08520404249429703, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.0593683123588562, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.03036445565521717, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.02010050229728222, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.18620601296424866, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.01005025114864111, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.5694770216941833, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.25949180126190186, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 2.2121424674987793, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.979899525642395, "metrics/particle/momentum/class_7/event/energy_bias": -1.296762466430664, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.1063125133514404, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 0.9157101511955261, "metrics/particle/momentum/class_7/event/px_bias": -0.12157727032899857, "metrics/particle/momentum/class_7/event/py_bias": -0.08845417201519012, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.7452794313430786, "metrics/particle/momentum/class_7/event/pz_bias": -0.25876742601394653, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.685422420501709, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 7.092079162597656, "step_valid_time": 24779.930265426636, "step_total_time": 24787.022344827652, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0266958885687638, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.34310109752760387, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81762, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15733, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13011, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19310217858238723, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19242435361170226, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19276267022795077, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8075756463882977, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15969315741024853, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15913260438834667, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.1594123881227908, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19310217858238723, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0255487528093312, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3623208682604935, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81699, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15375, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12455, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18925639163455976, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18819079792898322, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18872209061115272, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8118092020710168, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15331306392250027, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15244984638734868, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15288023665443298, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18925639163455976, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0255522331076925, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.38436491641156434, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81152, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 14471, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11755, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.17912091992721782, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.17831969637223974, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.17871941015555048, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8216803036277602, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14550248177350877, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14485163643533122, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14517632965092225, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.17912091992721782} \ No newline at end of file diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_20000.json b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..5050366240bf963773cd68c78f2b1a93b978c906 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 0.08618535021417059, "train": {"Classification_binary": 3.712048053741455, "Classification": 0.012214826419949532, "Regression_pt": 0.35306885838508606, "Regression_eta": 0.06459372490644455, "Regression_sin_phi": 0.05055144801735878, "Regression_cos_phi": 0.05100451782345772, "Regression_energy": 0.4060685336589813, "Total": 13.727483749389648, "Total_unweighted": 4.649549962952733}, "train_group": {"physical_pu": {"Classification_binary": 3.0039643271938785, "Classification": 0.015708541088215597, "Regression_pt": 0.23527124704240443, "Regression_eta": 0.10611172388870638, "Regression_sin_phi": 0.04760364578156681, "Regression_cos_phi": 0.0475616750235741, "Regression_energy": 0.28247367025731684, "Total": 12.0088591837621, "Total_unweighted": 3.7386948302756626}, "physical_nopu": {"Classification_binary": 3.5900470428865474, "Classification": 0.014748880891713372, "Regression_pt": 0.39199714664250973, "Regression_eta": 0.0703826219343444, "Regression_sin_phi": 0.04844972094555761, "Regression_cos_phi": 0.04868238559043971, "Regression_energy": 0.43952164919462927, "Total": 14.059256116155744, "Total_unweighted": 4.603829448085741}}, "valid": {"Classification_binary": 3.2879908879597983, "Classification": 0.015263209740320841, "Regression_pt": 0.3157092332839966, "Regression_eta": 0.08895671367645264, "Regression_sin_phi": 0.048401087522506714, "Regression_cos_phi": 0.048314765095710754, "Regression_energy": 0.36287176609039307, "Total": 13.084157307942709, "group/physical_pu/Classification": 0.01568051551779111, "group/physical_pu/Classification_binary": 2.987987836201986, "group/physical_pu/Regression_cos_phi": 0.04769455393155416, "group/physical_pu/Regression_energy": 0.2839886744817098, "group/physical_pu/Regression_eta": 0.10591380794843037, "group/physical_pu/Regression_pt": 0.23560728629430136, "group/physical_pu/Regression_sin_phi": 0.04762731989224752, "group/physical_pu/Total": 12.005448659261068, "group/physical_pu/Total_unweighted": 3.72449999426802, "group/physical_nopu/Classification": 0.014845906446377436, "group/physical_nopu/Classification_binary": 3.587993939717611, "group/physical_nopu/Regression_cos_phi": 0.048934986193974815, "group/physical_nopu/Regression_energy": 0.44175493717193604, "group/physical_nopu/Regression_eta": 0.07199963430563609, "group/physical_nopu/Regression_pt": 0.39581116040547687, "group/physical_nopu/Regression_sin_phi": 0.049174865086873375, "group/physical_nopu/Total": 14.16286595662435, "group/physical_nopu/Total_unweighted": 4.610515429327886, "Total_unweighted": 4.167507663369179, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.01501692645251751, "diagnostic/loss/cms_pf/Classification_binary": 3.276038646697998, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.0479571633040905, "diagnostic/loss/cms_pf/Regression_energy": 0.34844908118247986, "diagnostic/loss/cms_pf/Regression_eta": 0.08906224370002747, "diagnostic/loss/cms_pf/Regression_pt": 0.3014844059944153, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.04676641523838043, "diagnostic/regression/cms_pf/energy_pred_mean": 0.18113680183887482, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.3555019795894623, "diagnostic/regression/cms_pf/energy_residual_mean": 0.07037057727575302, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5855765342712402, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.10379692912101746, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.329225093126297, "diagnostic/regression/cms_pf/pt_residual_mean": 0.11456910520792007, "diagnostic/regression/cms_pf/pt_residual_rms": 0.546252429485321, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.889639675617218, "metrics/particle/class_1/pid_accuracy": 0.9774462580680847, "metrics/particle/class_2/efficiency": 0.2568356692790985, "metrics/particle/class_2/pid_accuracy": 0.20204032957553864, "metrics/particle/class_3/efficiency": 0.3909021019935608, "metrics/particle/class_3/pid_accuracy": 0.47549185156822205, "metrics/particle/class_4/efficiency": 0.4354071617126465, "metrics/particle/class_4/pid_accuracy": 0.6527107357978821, "metrics/particle/class_5/efficiency": 0.3790294826030731, "metrics/particle/class_5/pid_accuracy": 0.6287978887557983, "metrics/particle/class_6/efficiency": 0.8680742383003235, "metrics/particle/class_6/pid_accuracy": 0.05599085986614227, "metrics/particle/class_7/efficiency": 0.9638602137565613, "metrics/particle/class_7/pid_accuracy": 0.21878862380981445, "metrics/particle/count/bias_mean": 71.28255462646484, "metrics/particle/count/mae": 94.64974212646484, "metrics/particle/count/prediction_mean": 1599.8861083984375, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.10268963128328323, "metrics/particle/event/energy_response_mean": 0.9022531509399414, "metrics/particle/event/met_abs_error": 22.78617286682129, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04635841026902199, "metrics/particle/event/scalar_pt_response_mean": 0.9969335198402405, "metrics/particle/event/vector_pt_closure": 0.030815288424491882, "metrics/particle/matched/delta_eta_abs_mean": 0.019025690853595734, "metrics/particle/matched/delta_phi_abs_mean": 0.01717557944357395, "metrics/particle/matched/delta_r_mean": 0.028601091355085373, "metrics/particle/matched/energy_relative_abs_mean": 0.11338480561971664, "metrics/particle/matched/pid_accuracy": 0.844409704208374, "metrics/particle/matched/pt_relative_abs_mean": 0.10570640861988068, "metrics/particle/matching/duplicate_fraction": 0.11366898566484451, "metrics/particle/matching/efficiency": 0.6354952454566956, "metrics/particle/matching/f1": 0.6210154891014099, "metrics/particle/matching/purity": 0.6071808934211731, "metrics/particle/matching/target_coverage_dr0p05": 0.5163886547088623, "metrics/particle/matching/target_coverage_dr0p10": 0.7009367942810059, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5555509328842163, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7537246346473694, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018218321725726128, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06580403447151184, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05837329104542732, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0001844215003075078, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.10885452479124069, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0018104040063917637, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.21710141003131866, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.014318578876554966, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.3792376518249512, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68493.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4682866930961609, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1457548141479492, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.3819941282272339, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.2535320520401001, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1389598846435547, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 3.1897695064544678, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03872483968734741, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.809154987335205, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.12247925996780396, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.153358459472656, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87644.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7814286351203918, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 30.512908935546875, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 42.144187927246094, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 16.39657211303711, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.6133747100830078, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.13683602213859558, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 36.43561935424805, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.7725114822387695, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 55.12833023071289, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04117726907134056, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16163910925388336, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15992940962314606, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.017404770478606224, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2133142650127411, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17098183929920197, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.4305033683776855, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.03263508528470993, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.627225875854492, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 120621.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5396894216537476, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9227805733680725, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9333962798118591, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7102870345115662, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3221276104450226, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3.7945032119750977, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.3771960139274597, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.901065826416016, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.18630553781986237, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.81434631347656, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228425.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5214667916297913, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -502.35089111328125, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 424.62542724609375, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 20.224857330322266, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.3226322531700134, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -4.521726608276367, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 423.0547790527344, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -208.95016479492188, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 625.0406494140625, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014904686249792576, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.06975918263196945, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.04986165091395378, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.012216608040034771, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10599762946367264, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006376694655045867, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.07652024179697037, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.007462496403604746, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6918665766716003, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59592.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.33321818709373474, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3329439163208008, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.268419086933136, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.1724993884563446, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10641568899154663, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 3.531003713607788, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.022299787029623985, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0459699630737305, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.07753156125545502, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.517522811889648, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72198.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8253968358039856, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -3.6980323791503906, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 23.097536087036133, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 17.209592819213867, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 2.678340435028076, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -3.4352989196777344, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 12.725351333618164, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.0446398258209229, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 23.729713439941406, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.028559306636452675, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11323117464780807, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10558797419071198, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.0054632266983389854, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1674039214849472, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08357658982276917, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.7118871212005615, "metrics/particle/momentum/all/accepted/pz_bias": 0.01798306033015251, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.136714935302734, "metrics/particle/momentum/all/accepted/samples": 248706.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0734978914260864, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0493824481964111, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6852755546569824, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5071818232536316, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.24066942930221558, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.617446184158325, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.23479975759983063, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.957785606384277, "metrics/particle/momentum/all/assigned/pz_bias": -0.15167149901390076, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.69635772705078, "metrics/particle/momentum/all/assigned/samples": 388267.0, "metrics/particle/momentum/all/efficiency": 0.6358165740966797, "metrics/particle/momentum/all/event/energy_bias": -475.5359802246094, "metrics/particle/momentum/all/event/p3_error_mean": 419.6893310546875, "metrics/particle/momentum/all/event/pt_vector_error_mean": 30.102266311645508, "metrics/particle/momentum/all/event/px_bias": 3.969082832336426, "metrics/particle/momentum/all/event/py_bias": -7.820189952850342, "metrics/particle/momentum/all/event/pz_abs_mean": 414.7177734375, "metrics/particle/momentum/all/event/pz_bias": -212.76730346679688, "metrics/particle/momentum/all/event/pz_residual_rms": 622.2341918945312, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01517587061971426, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06091313809156418, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05189884454011917, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007490713149309158, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09518790990114212, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011774924350902438, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.2015976458787918, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0028120651841163635, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.033030390739441, "metrics/particle/momentum/class_1/accepted/samples": 171551.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.22307603061199188, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.8946564793586731, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3116016983985901, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2526301145553589, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0772247239947319, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.481562376022339, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.022439155727624893, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1201814413070679, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0009037653217092156, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.408450603485107, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8898427486419678, "metrics/particle/momentum/class_1/event/energy_bias": 23.354812622070312, "metrics/particle/momentum/class_1/event/p3_error_mean": 57.509422302246094, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 18.33152198791504, "metrics/particle/momentum/class_1/event/px_bias": -3.9116384983062744, "metrics/particle/momentum/class_1/event/py_bias": -1.1150479316711426, "metrics/particle/momentum/class_1/event/pz_abs_mean": 50.902320861816406, "metrics/particle/momentum/class_1/event/pz_bias": 5.031618595123291, "metrics/particle/momentum/class_1/event/pz_residual_rms": 81.46648406982422, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.05795107036828995, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.24237142503261566, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23320235311985016, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07644771039485931, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2734253406524658, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.109333336353302, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.027719974517822, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.028231170028448105, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 9.257034301757812, "metrics/particle/momentum/class_2/accepted/samples": 7875.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.2062153816223145, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.4758058786392212, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0951017141342163, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5927956104278564, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4752520024776459, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.8853402137756348, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.3545551300048828, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.663259506225586, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.18196605145931244, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 34.972015380859375, "metrics/particle/momentum/class_2/assigned/samples": 31053.0, "metrics/particle/momentum/class_2/efficiency": 0.2534925639629364, "metrics/particle/momentum/class_2/event/energy_bias": -337.61981201171875, "metrics/particle/momentum/class_2/event/p3_error_mean": 170.53451538085938, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 29.53888702392578, "metrics/particle/momentum/class_2/event/px_bias": 3.113008737564087, "metrics/particle/momentum/class_2/event/py_bias": -6.353838920593262, "metrics/particle/momentum/class_2/event/pz_abs_mean": 164.20635986328125, "metrics/particle/momentum/class_2/event/pz_bias": -37.60105895996094, "metrics/particle/momentum/class_2/event/pz_residual_rms": 230.5181121826172, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.06076769530773163, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.2275959849357605, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.22464619576931, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.008828343823552132, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.26418784260749817, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.3093978464603424, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.262361526489258, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.09807591885328293, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 13.786524772644043, "metrics/particle/momentum/class_3/accepted/samples": 20792.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.071763753890991, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.9205203652381897, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.066530704498291, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.8820648789405823, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4239772856235504, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 2.9144127368927, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5350375771522522, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.546245574951172, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.18037182092666626, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.95970916748047, "metrics/particle/momentum/class_3/assigned/samples": 52458.0, "metrics/particle/momentum/class_3/efficiency": 0.39008650183677673, "metrics/particle/momentum/class_3/event/energy_bias": 507.6568603515625, "metrics/particle/momentum/class_3/event/p3_error_mean": 309.9186706542969, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.74286937713623, "metrics/particle/momentum/class_3/event/px_bias": 0.29926320910453796, "metrics/particle/momentum/class_3/event/py_bias": -1.5075294971466064, "metrics/particle/momentum/class_3/event/pz_abs_mean": 309.5173645019531, "metrics/particle/momentum/class_3/event/pz_bias": -157.71878051757812, "metrics/particle/momentum/class_3/event/pz_residual_rms": 494.5919189453125, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.059860143810510635, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.23278877139091492, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22982430458068848, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.058491457253694534, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2687681019306183, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.3538716733455658, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 10.895244598388672, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.04583203047513962, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.57215690612793, "metrics/particle/momentum/class_4/accepted/samples": 36793.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.8863892555236816, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.9927967190742493, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0682252645492554, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.7842985987663269, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3704865276813507, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.897859811782837, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5286551117897034, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.703340530395508, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.44237005710601807, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 40.95040512084961, "metrics/particle/momentum/class_4/assigned/samples": 82254.0, "metrics/particle/momentum/class_4/efficiency": 0.4365359842777252, "metrics/particle/momentum/class_4/event/energy_bias": -603.3548583984375, "metrics/particle/momentum/class_4/event/p3_error_mean": 332.1457214355469, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 11.45737075805664, "metrics/particle/momentum/class_4/event/px_bias": -0.44094690680503845, "metrics/particle/momentum/class_4/event/py_bias": -0.7876483201980591, "metrics/particle/momentum/class_4/event/pz_abs_mean": 331.56219482421875, "metrics/particle/momentum/class_4/event/pz_bias": -13.12031078338623, "metrics/particle/momentum/class_4/event/pz_residual_rms": 489.57769775390625, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.05143549293279648, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22266535460948944, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21250320971012115, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.028952207416296005, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2534562051296234, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02354331500828266, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2907705307006836, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.01099619921296835, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.11156702041626, "metrics/particle/momentum/class_5/accepted/samples": 11001.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4257454872131348, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.0218725204467773, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9616175889968872, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6490060687065125, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.3809260427951813, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.460742473602295, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.1468210071325302, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.4338059425354, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.2571397125720978, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 17.9481201171875, "metrics/particle/momentum/class_5/assigned/samples": 28940.0, "metrics/particle/momentum/class_5/efficiency": 0.38003939390182495, "metrics/particle/momentum/class_5/event/energy_bias": -56.059814453125, "metrics/particle/momentum/class_5/event/p3_error_mean": 109.5975341796875, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 25.181087493896484, "metrics/particle/momentum/class_5/event/px_bias": 5.008795261383057, "metrics/particle/momentum/class_5/event/py_bias": 2.362236261367798, "metrics/particle/momentum/class_5/event/pz_abs_mean": 101.99505615234375, "metrics/particle/momentum/class_5/event/pz_bias": -7.190923690795898, "metrics/particle/momentum/class_5/event/pz_residual_rms": 158.60894775390625, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.017208276316523552, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.0990673378109932, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.09896925836801529, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.05382081866264343, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.14317099750041962, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.015968063846230507, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.8166154623031616, "metrics/particle/momentum/class_6/accepted/pz_bias": 0.43495839834213257, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 3.7481002807617188, "metrics/particle/momentum/class_6/accepted/samples": 501.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.29662570357322693, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 0.9943495392799377, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.5694829821586609, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.3995395004749298, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.09565217047929764, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 2.6287097930908203, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.043478261679410934, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 2.2155661582946777, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.20643477141857147, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 9.866344451904297, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8713043332099915, "metrics/particle/momentum/class_6/event/energy_bias": -7.799256324768066, "metrics/particle/momentum/class_6/event/p3_error_mean": 9.325769424438477, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 3.4179556369781494, "metrics/particle/momentum/class_6/event/px_bias": 0.006978623569011688, "metrics/particle/momentum/class_6/event/py_bias": -0.35031595826148987, "metrics/particle/momentum/class_6/event/pz_abs_mean": 8.265774726867676, "metrics/particle/momentum/class_6/event/pz_bias": -2.0253851413726807, "metrics/particle/momentum/class_6/event/pz_residual_rms": 25.936904907226562, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.013939649797976017, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.033903662115335464, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.033656638115644455, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.012151086702942848, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.059486761689186096, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.015544041059911251, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.5153151750564575, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.28329670429229736, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 1.7866724729537964, "metrics/particle/momentum/class_7/accepted/samples": 193.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.06306076794862747, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.4946011006832123, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.10087619721889496, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.07439831644296646, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.02010050229728222, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.76244056224823, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.02010050229728222, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.7729958891868591, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.018628723919391632, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 3.492995500564575, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9698492288589478, "metrics/particle/momentum/class_7/event/energy_bias": -1.7139925956726074, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.2503011226654053, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 1.044833779335022, "metrics/particle/momentum/class_7/event/px_bias": -0.10637819021940231, "metrics/particle/momentum/class_7/event/py_bias": -0.06804538518190384, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.7850470542907715, "metrics/particle/momentum/class_7/event/pz_bias": -0.14246545732021332, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.71637487411499, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 6.174542427062988, "step_valid_time": 4819.828399658203, "step_total_time": 4826.002942562103, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0354541863246678, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3678177114406159, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84519, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16418, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13185, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.20150966554157718, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19425217998319905, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19781437883296987, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.805747820016801, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1618287818349187, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15600042593972951, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.1588611636565177, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.20150966554157718, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0306368382803304, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3882675704705558, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 84395, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15582, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12370, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.1918044289072982, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18463179098287813, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18814977601217142, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8153682090171218, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1522667684240328, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1465726642573612, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14936546844246953, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.1918044289072982, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.034660430852004, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.4007298409464549, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 83871, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15338, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12185, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18985257894020224, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18287608350919865, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18629904044698167, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8171239164908013, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15082498855042148, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14528263642975522, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14800194339851816, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18985257894020224} \ No newline at end of file diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_30000.json b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..915671dfa639b8fa1845e0203de71be2e7832bf8 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 0.1292780253212559, "train": {"Classification_binary": 3.598886728286743, "Classification": 0.012575743719935417, "Regression_pt": 0.3290678858757019, "Regression_eta": 0.07072009146213531, "Regression_sin_phi": 0.0494237020611763, "Regression_cos_phi": 0.04818630591034889, "Regression_energy": 0.3864896595478058, "Total": 13.291566848754883, "Total_unweighted": 4.495350116863847}, "train_group": {"physical_pu": {"Classification_binary": 2.9694393309918077, "Classification": 0.01565375470730302, "Regression_pt": 0.2339820948603389, "Regression_eta": 0.10595653459920988, "Regression_sin_phi": 0.04737683000800374, "Regression_cos_phi": 0.04743118399938384, "Regression_energy": 0.28061562350818087, "Total": 11.937054172976987, "Total_unweighted": 3.7004553526742283}, "physical_nopu": {"Classification_binary": 3.5392997889104323, "Classification": 0.014637669347579067, "Regression_pt": 0.3850514152557543, "Regression_eta": 0.07016933228250909, "Regression_sin_phi": 0.048340490434167147, "Regression_cos_phi": 0.04849510592366471, "Regression_energy": 0.4317622460810134, "Total": 13.899034315472258, "Total_unweighted": 4.53775604823512}}, "valid": {"Classification_binary": 3.249126752217611, "Classification": 0.015123774607976278, "Regression_pt": 0.30941704909006756, "Regression_eta": 0.08853091796239217, "Regression_sin_phi": 0.04811850190162659, "Regression_cos_phi": 0.04806153476238251, "Regression_energy": 0.35594205061594647, "Total": 12.931382497151693, "group/physical_pu/Classification": 0.015623963127533594, "group/physical_pu/Classification_binary": 2.960186322530111, "group/physical_pu/Regression_cos_phi": 0.0475521981716156, "group/physical_pu/Regression_energy": 0.27940640846888226, "group/physical_pu/Regression_eta": 0.10544952750205994, "group/physical_pu/Regression_pt": 0.23206621408462524, "group/physical_pu/Regression_sin_phi": 0.047485753893852234, "group/physical_pu/Total": 11.907828013102213, "group/physical_pu/Total_unweighted": 3.68777038777868, "group/physical_nopu/Classification": 0.014623584846655527, "group/physical_nopu/Classification_binary": 3.538066864013672, "group/physical_nopu/Regression_cos_phi": 0.048570881287256874, "group/physical_nopu/Regression_energy": 0.4324777126312256, "group/physical_nopu/Regression_eta": 0.07161231338977814, "group/physical_nopu/Regression_pt": 0.3867679039637248, "group/physical_nopu/Regression_sin_phi": 0.048751244942347206, "group/physical_nopu/Total": 13.954936981201172, "group/physical_nopu/Total_unweighted": 4.54087050507466, "Total_unweighted": 4.114320581158003, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.014879628084599972, "diagnostic/loss/cms_pf/Classification_binary": 3.2415432929992676, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.0477796271443367, "diagnostic/loss/cms_pf/Regression_energy": 0.3428308665752411, "diagnostic/loss/cms_pf/Regression_eta": 0.08853884041309357, "diagnostic/loss/cms_pf/Regression_pt": 0.29635345935821533, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.04642893373966217, "diagnostic/regression/cms_pf/energy_pred_mean": 0.20863613486289978, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.35715237259864807, "diagnostic/regression/cms_pf/energy_residual_mean": 0.09786991029977798, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5903400778770447, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.1213778629899025, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.3302028179168701, "diagnostic/regression/cms_pf/pt_residual_mean": 0.1321500539779663, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5504291653633118, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8906751275062561, "metrics/particle/class_1/pid_accuracy": 0.9779282212257385, "metrics/particle/class_2/efficiency": 0.2561793029308319, "metrics/particle/class_2/pid_accuracy": 0.2040952891111374, "metrics/particle/class_3/efficiency": 0.38252586126327515, "metrics/particle/class_3/pid_accuracy": 0.47121626138687134, "metrics/particle/class_4/efficiency": 0.43101561069488525, "metrics/particle/class_4/pid_accuracy": 0.664130687713623, "metrics/particle/class_5/efficiency": 0.37844619154930115, "metrics/particle/class_5/pid_accuracy": 0.6338408589363098, "metrics/particle/class_6/efficiency": 0.8655235767364502, "metrics/particle/class_6/pid_accuracy": 0.06434184312820435, "metrics/particle/class_7/efficiency": 0.9610801935195923, "metrics/particle/class_7/pid_accuracy": 0.25206610560417175, "metrics/particle/count/bias_mean": 63.27994918823242, "metrics/particle/count/mae": 84.04166412353516, "metrics/particle/count/prediction_mean": 1591.8834228515625, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.08539960533380508, "metrics/particle/event/energy_response_mean": 0.9531511664390564, "metrics/particle/event/met_abs_error": 21.8902587890625, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05180317535996437, "metrics/particle/event/scalar_pt_response_mean": 1.0132060050964355, "metrics/particle/event/vector_pt_closure": 0.029731683433055878, "metrics/particle/matched/delta_eta_abs_mean": 0.02045215107500553, "metrics/particle/matched/delta_phi_abs_mean": 0.01699022948741913, "metrics/particle/matched/delta_r_mean": 0.029754487797617912, "metrics/particle/matched/energy_relative_abs_mean": 0.1130371019244194, "metrics/particle/matched/pid_accuracy": 0.8475376963615417, "metrics/particle/matched/pt_relative_abs_mean": 0.10433486104011536, "metrics/particle/matching/duplicate_fraction": 0.11120744794607162, "metrics/particle/matching/efficiency": 0.6338058710098267, "metrics/particle/matching/f1": 0.6209530234336853, "metrics/particle/matching/purity": 0.6086110472679138, "metrics/particle/matching/target_coverage_dr0p05": 0.511264979839325, "metrics/particle/matching/target_coverage_dr0p10": 0.6981770992279053, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5562819838523865, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7525521516799927, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019958775490522385, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06544134020805359, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05762169882655144, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0007904689409770072, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.10965195298194885, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0019861843902617693, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.21996617317199707, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.01311664842069149, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5294644832611084, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68473.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.47220227122306824, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1596901416778564, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.3795278072357178, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.25423526763916016, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13878557085990906, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.3042423725128174, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.04026424512267113, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8729825019836426, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.09145829826593399, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.517094612121582, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87646.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7812004685401917, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 36.469703674316406, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 44.71859359741211, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 16.236984252929688, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 2.2494659423828125, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.42899489402770996, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 39.77073669433594, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 3.4823222160339355, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 62.0849494934082, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04229521006345749, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1606869399547577, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15812967717647552, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.012953408993780613, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21140819787979126, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17390431463718414, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.4819793701171875, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.02009180188179016, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.630239486694336, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 119629.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5431838035583496, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9859858751296997, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.967174232006073, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7581493854522705, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.32598915696144104, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3.756340742111206, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.38254621624946594, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.89511775970459, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.02529633603990078, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.42282485961914, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228707.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5171781778335571, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 493.8761901855469, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 381.840576171875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 20.055458068847656, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.6917357444763184, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 3.8784985542297363, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 379.66729736328125, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -33.61317825317383, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 562.8362426757812, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01549530029296875, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.07029744982719421, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.04832841455936432, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010048244148492813, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10537998378276825, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000636697222944349, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.07709962874650955, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.004667073953896761, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.722538411617279, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59683.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.33499276638031006, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.4437859058380127, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.26470237970352173, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.16981929540634155, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10396553575992584, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.527270793914795, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.022563263773918152, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0769586563110352, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.07837692648172379, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.668187618255615, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72197.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.826657235622406, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -7.706032752990723, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 22.426685333251953, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 17.01494598388672, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 3.964681625366211, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -3.9893672466278076, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 12.510581016540527, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.3278264999389648, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 26.58766746520996, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02966756373643875, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11259499192237854, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10390786826610565, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.0036151029635220766, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1660574972629547, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08466210961341858, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.726020574569702, "metrics/particle/momentum/all/accepted/pz_bias": -0.004951392766088247, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.128716468811035, "metrics/particle/momentum/all/accepted/samples": 247785.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0771046876907349, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.110233187675476, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7040903568267822, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5351620316505432, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.24250675737857819, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.64849853515625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.23844808340072632, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.97873592376709, "metrics/particle/momentum/all/assigned/pz_bias": -0.050083599984645844, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.445436477661133, "metrics/particle/momentum/all/assigned/samples": 388550.0, "metrics/particle/momentum/all/efficiency": 0.6334620118141174, "metrics/particle/momentum/all/event/energy_bias": 522.639892578125, "metrics/particle/momentum/all/event/p3_error_mean": 383.4188537597656, "metrics/particle/momentum/all/event/pt_vector_error_mean": 27.10913848876953, "metrics/particle/momentum/all/event/px_bias": 6.905882835388184, "metrics/particle/momentum/all/event/py_bias": 0.31812605261802673, "metrics/particle/momentum/all/event/pz_abs_mean": 379.4022216796875, "metrics/particle/momentum/all/event/pz_bias": -28.803037643432617, "metrics/particle/momentum/all/event/pz_residual_rms": 560.4739990234375, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.016911184415221214, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06198659911751747, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.051569484174251556, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006960379425436258, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09654869139194489, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012283653486520052, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.20568208396434784, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.005723756738007069, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.0349186658859253, "metrics/particle/momentum/class_1/accepted/samples": 171773.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.22575919330120087, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.9361366033554077, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.31682512164115906, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2586418688297272, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0762910544872284, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.857475519180298, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.022921551018953323, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.160481572151184, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.015821315348148346, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.718231678009033, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8909942507743835, "metrics/particle/momentum/class_1/event/energy_bias": 52.06703567504883, "metrics/particle/momentum/class_1/event/p3_error_mean": 56.429344177246094, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 16.502796173095703, "metrics/particle/momentum/class_1/event/px_bias": 0.9845253229141235, "metrics/particle/momentum/class_1/event/py_bias": -1.6237014532089233, "metrics/particle/momentum/class_1/event/pz_abs_mean": 50.321495056152344, "metrics/particle/momentum/class_1/event/pz_bias": -5.69014310836792, "metrics/particle/momentum/class_1/event/pz_residual_rms": 92.88273620605469, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.05818217247724533, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2395266890525818, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23076577484607697, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06815093010663986, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2709585726261139, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.10986632853746414, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.227971076965332, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.058241114020347595, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 10.860901832580566, "metrics/particle/momentum/class_2/accepted/samples": 7855.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.2005860805511475, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.546486496925354, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1294927597045898, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6478952765464783, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.47481968998908997, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.6610896587371826, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.3552936613559723, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.72032928466797, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.05463471636176109, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 35.032291412353516, "metrics/particle/momentum/class_2/assigned/samples": 31056.0, "metrics/particle/momentum/class_2/efficiency": 0.2528487741947174, "metrics/particle/momentum/class_2/event/energy_bias": -238.4049835205078, "metrics/particle/momentum/class_2/event/p3_error_mean": 169.28219604492188, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 29.929508209228516, "metrics/particle/momentum/class_2/event/px_bias": 4.098757266998291, "metrics/particle/momentum/class_2/event/py_bias": -3.6259801387786865, "metrics/particle/momentum/class_2/event/pz_abs_mean": 163.0624542236328, "metrics/particle/momentum/class_2/event/pz_bias": -0.12541818618774414, "metrics/particle/momentum/class_2/event/pz_residual_rms": 227.13009643554688, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.060891781002283096, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.225469708442688, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.22238269448280334, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.001723185763694346, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2622373700141907, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.31891652941703796, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.480767250061035, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.050205133855342865, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 14.185379028320312, "metrics/particle/momentum/class_3/accepted/samples": 20416.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.084955930709839, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.0071998834609985, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.1271895170211792, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.9582934379577637, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4338439702987671, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 2.9081366062164307, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5474067330360413, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.808313369750977, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.008782717399299145, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.876007080078125, "metrics/particle/momentum/class_3/assigned/samples": 52482.0, "metrics/particle/momentum/class_3/efficiency": 0.3830322027206421, "metrics/particle/momentum/class_3/event/energy_bias": 550.2135009765625, "metrics/particle/momentum/class_3/event/p3_error_mean": 293.1372985839844, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.896830558776855, "metrics/particle/momentum/class_3/event/px_bias": 0.09001700580120087, "metrics/particle/momentum/class_3/event/py_bias": 1.626951813697815, "metrics/particle/momentum/class_3/event/pz_abs_mean": 292.795166015625, "metrics/particle/momentum/class_3/event/pz_bias": 42.881099700927734, "metrics/particle/momentum/class_3/event/pz_residual_rms": 468.3177795410156, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.06014152243733406, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.22979097068309784, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22658264636993408, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.050667695701122284, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2656336724758148, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.3636363744735718, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 11.0524263381958, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.0431148037314415, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.366147994995117, "metrics/particle/momentum/class_4/accepted/samples": 36091.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.886472463607788, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0745598077774048, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0873382091522217, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8252811431884766, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.37535905838012695, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.383775472640991, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5349857807159424, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.443029403686523, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.19807854294776917, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 40.000816345214844, "metrics/particle/momentum/class_4/assigned/samples": 82505.0, "metrics/particle/momentum/class_4/efficiency": 0.42820701003074646, "metrics/particle/momentum/class_4/event/energy_bias": 201.63194274902344, "metrics/particle/momentum/class_4/event/p3_error_mean": 322.8829040527344, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 11.368895530700684, "metrics/particle/momentum/class_4/event/px_bias": -0.10914187133312225, "metrics/particle/momentum/class_4/event/py_bias": 1.8065825700759888, "metrics/particle/momentum/class_4/event/pz_abs_mean": 321.99774169921875, "metrics/particle/momentum/class_4/event/pz_bias": -72.9256820678711, "metrics/particle/momentum/class_4/event/pz_residual_rms": 483.34375, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.05134652554988861, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2209566980600357, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21043157577514648, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.029763761907815933, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.25173306465148926, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.024005111306905746, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2850464582443237, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.01981731317937374, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.930764675140381, "metrics/particle/momentum/class_5/accepted/samples": 10956.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4291589260101318, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.094797372817993, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9729859232902527, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6661023497581482, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.3790982961654663, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.1248931884765625, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.14859215915203094, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.533528804779053, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.251905620098114, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 18.28407859802246, "metrics/particle/momentum/class_5/assigned/samples": 28945.0, "metrics/particle/momentum/class_5/efficiency": 0.3784848153591156, "metrics/particle/momentum/class_5/event/energy_bias": -32.700191497802734, "metrics/particle/momentum/class_5/event/p3_error_mean": 113.82634735107422, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 26.234651565551758, "metrics/particle/momentum/class_5/event/px_bias": 2.5505895614624023, "metrics/particle/momentum/class_5/event/py_bias": 3.2010369300842285, "metrics/particle/momentum/class_5/event/pz_abs_mean": 105.89729309082031, "metrics/particle/momentum/class_5/event/pz_bias": 7.094777584075928, "metrics/particle/momentum/class_5/event/pz_residual_rms": 167.14279174804688, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.01980467513203621, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.09015162289142609, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.09053394198417664, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.03992925584316254, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.1329275369644165, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.009980039671063423, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.6084203720092773, "metrics/particle/momentum/class_6/accepted/pz_bias": 0.30640506744384766, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 2.33925724029541, "metrics/particle/momentum/class_6/accepted/samples": 501.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.35448524355888367, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 1.2418854236602783, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.6518508791923523, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.4972821772098541, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.09913043677806854, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 3.2846076488494873, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.04521739110350609, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 2.2759978771209717, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.16037295758724213, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 10.075287818908691, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8713043332099915, "metrics/particle/momentum/class_6/event/energy_bias": -8.902847290039062, "metrics/particle/momentum/class_6/event/p3_error_mean": 7.3417744636535645, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 2.799139976501465, "metrics/particle/momentum/class_6/event/px_bias": -0.5505685806274414, "metrics/particle/momentum/class_6/event/py_bias": -0.9695761203765869, "metrics/particle/momentum/class_6/event/pz_abs_mean": 6.4266886711120605, "metrics/particle/momentum/class_6/event/pz_bias": -0.0525016188621521, "metrics/particle/momentum/class_6/event/pz_residual_rms": 17.917699813842773, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.015866056084632874, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.03988666832447052, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.03779926151037216, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.008773923851549625, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.07385194301605225, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.005181347019970417, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.4630489945411682, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.11864025890827179, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 1.339611291885376, "metrics/particle/momentum/class_7/accepted/samples": 193.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.03552214428782463, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.06593437492847443, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.054358627647161484, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.019488733261823654, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.02010050229728222, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.13927237689495087, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.005025125574320555, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.4952835440635681, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.0997428223490715, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 1.3715965747833252, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9698492288589478, "metrics/particle/momentum/class_7/event/energy_bias": -1.2645798921585083, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.1794633865356445, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 0.9549875855445862, "metrics/particle/momentum/class_7/event/px_bias": -0.15829549729824066, "metrics/particle/momentum/class_7/event/py_bias": -0.09718795120716095, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.7875041961669922, "metrics/particle/momentum/class_7/event/pz_bias": 0.014826085418462753, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.770107269287109, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 3.382559061050415, "step_valid_time": 7307.604739904404, "step_total_time": 7310.987299203873, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.026764204304289, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3621716188215749, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 82413, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16060, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12999, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.1971156796563363, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19487216822588668, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19598750366103682, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8051278317741133, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15954587296716785, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15772996978632012, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15863272478765986, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.1971156796563363, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0271351062900131, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.37390214119402754, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 82305, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15619, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12583, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19225987518310172, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18976975882388677, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19100670156043634, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8102302411761132, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15488866184960426, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15288257092521718, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15387907841314877, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19225987518310172, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0242512659625738, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3951698321248658, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81749, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 14811, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11880, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18332941365779995, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18117652815324958, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.1822466130997059, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8188234718467504, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14704972211563455, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14532287856732193, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14618120070383542, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18332941365779995} \ No newline at end of file diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_40000.json b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..6efb14c113cb213f404da573847131f51a3b9b3f --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 0.17237070042834118, "train": {"Classification_binary": 3.553797721862793, "Classification": 0.012087363749742508, "Regression_pt": 0.5255565047264099, "Regression_eta": 0.07435477524995804, "Regression_sin_phi": 0.04617923125624657, "Regression_cos_phi": 0.04603924602270126, "Regression_energy": 0.5661100745201111, "Total": 15.446720123291016, "Total_unweighted": 4.824124917387962}, "train_group": {"physical_pu": {"Classification_binary": 2.95189190696884, "Classification": 0.01566243136895227, "Regression_pt": 0.23058817314577626, "Regression_eta": 0.10536755179310893, "Regression_sin_phi": 0.047258971320403804, "Regression_cos_phi": 0.04724916643821276, "Regression_energy": 0.2773347808109535, "Total": 11.857306386088277, "Total_unweighted": 3.6753529818462476}, "physical_nopu": {"Classification_binary": 3.5153772691951204, "Classification": 0.014478923844089537, "Regression_pt": 0.380671560518419, "Regression_eta": 0.06987445383067309, "Regression_sin_phi": 0.0482955081275653, "Regression_cos_phi": 0.04827995818800921, "Regression_energy": 0.42748753406820517, "Total": 13.80153357733463, "Total_unweighted": 4.504465207772082}}, "valid": {"Classification_binary": 3.230462074279785, "Classification": 0.01508138949672381, "Regression_pt": 0.3052499492963155, "Regression_eta": 0.08833606044451396, "Regression_sin_phi": 0.04800131916999817, "Regression_cos_phi": 0.047846208016077675, "Regression_energy": 0.35114137331644696, "Total": 12.84005610148112, "group/physical_pu/Classification": 0.015610170861085257, "group/physical_pu/Classification_binary": 2.94445006052653, "group/physical_pu/Regression_cos_phi": 0.04747249186038971, "group/physical_pu/Regression_energy": 0.27753837903340656, "group/physical_pu/Regression_eta": 0.10535004734992981, "group/physical_pu/Regression_pt": 0.2308508555094401, "group/physical_pu/Regression_sin_phi": 0.04741450150807699, "group/physical_pu/Total": 11.865564982096354, "group/physical_pu/Total_unweighted": 3.668686506648858, "group/physical_nopu/Classification": 0.014552608132362366, "group/physical_nopu/Classification_binary": 3.5164737701416016, "group/physical_nopu/Regression_cos_phi": 0.04821992417176565, "group/physical_nopu/Regression_energy": 0.42474432786305744, "group/physical_nopu/Regression_eta": 0.0713220586379369, "group/physical_nopu/Regression_pt": 0.3796490828196208, "group/physical_nopu/Regression_sin_phi": 0.04858813683191935, "group/physical_nopu/Total": 13.814547220865885, "group/physical_nopu/Total_unweighted": 4.503549908598264, "Total_unweighted": 4.086118374019861, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.014923138543963432, "diagnostic/loss/cms_pf/Classification_binary": 3.2272870540618896, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.04761795327067375, "diagnostic/loss/cms_pf/Regression_energy": 0.33885177969932556, "diagnostic/loss/cms_pf/Regression_eta": 0.08830665051937103, "diagnostic/loss/cms_pf/Regression_pt": 0.29316139221191406, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.04630162939429283, "diagnostic/regression/cms_pf/energy_pred_mean": 0.1970134973526001, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.3537450432777405, "diagnostic/regression/cms_pf/energy_residual_mean": 0.08624725788831711, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5843507051467896, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.10726882517337799, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.3266437351703644, "diagnostic/regression/cms_pf/pt_residual_mean": 0.1180410236120224, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5440053343772888, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8933039307594299, "metrics/particle/class_1/pid_accuracy": 0.9780735373497009, "metrics/particle/class_2/efficiency": 0.25624626874923706, "metrics/particle/class_2/pid_accuracy": 0.20171046257019043, "metrics/particle/class_3/efficiency": 0.40096044540405273, "metrics/particle/class_3/pid_accuracy": 0.48168209195137024, "metrics/particle/class_4/efficiency": 0.4392002522945404, "metrics/particle/class_4/pid_accuracy": 0.6585404276847839, "metrics/particle/class_5/efficiency": 0.36941853165626526, "metrics/particle/class_5/pid_accuracy": 0.6275667548179626, "metrics/particle/class_6/efficiency": 0.8665155172348022, "metrics/particle/class_6/pid_accuracy": 0.05903515964746475, "metrics/particle/class_7/efficiency": 0.9638602137565613, "metrics/particle/class_7/pid_accuracy": 0.21961268782615662, "metrics/particle/count/bias_mean": 67.9287109375, "metrics/particle/count/mae": 92.52962493896484, "metrics/particle/count/prediction_mean": 1596.5322265625, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.0870075523853302, "metrics/particle/event/energy_response_mean": 0.9283294677734375, "metrics/particle/event/met_abs_error": 20.44266700744629, "metrics/particle/event/scalar_pt_relative_abs_error": 0.047316331416368484, "metrics/particle/event/scalar_pt_response_mean": 1.0001057386398315, "metrics/particle/event/vector_pt_closure": 0.029094725847244263, "metrics/particle/matched/delta_eta_abs_mean": 0.020349236205220222, "metrics/particle/matched/delta_phi_abs_mean": 0.016359593719244003, "metrics/particle/matched/delta_r_mean": 0.02931242436170578, "metrics/particle/matched/energy_relative_abs_mean": 0.11432185769081116, "metrics/particle/matched/pid_accuracy": 0.845805823802948, "metrics/particle/matched/pt_relative_abs_mean": 0.10565400868654251, "metrics/particle/matching/duplicate_fraction": 0.10994083434343338, "metrics/particle/matching/efficiency": 0.6387508511543274, "metrics/particle/matching/f1": 0.6248668432235718, "metrics/particle/matching/purity": 0.6115735173225403, "metrics/particle/matching/target_coverage_dr0p05": 0.5135359168052673, "metrics/particle/matching/target_coverage_dr0p10": 0.7014536261558533, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5577102899551392, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7579278945922852, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019290992990136147, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06624231487512589, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05852288007736206, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0011141861323267221, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.11085782945156097, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.001744312816299498, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.22092510759830475, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.004066463094204664, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.3664237260818481, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68795.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.46922340989112854, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.080708622932434, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.3647301495075226, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.2397920936346054, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13427811861038208, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.0793347358703613, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.039799634367227554, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8117592334747314, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.061270590871572495, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.117276191711426, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87639.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7848740816116333, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 30.64065933227539, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 43.396724700927734, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 16.35540771484375, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.6036803722381592, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.6817108392715454, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 38.29018783569336, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 3.6525471210479736, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 59.47452926635742, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04201699420809746, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16273774206638336, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1595645397901535, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.015597491525113583, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2129542976617813, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17304764688014984, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.48077917098999, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.004942670930176973, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.633903503417969, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 121533.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5105491876602173, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9501897692680359, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9304806590080261, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.709868311882019, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.31572723388671875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 4.319659233093262, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.3750629723072052, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.731406211853027, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.09695503860712051, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.31174087524414, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228311.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5254095196723938, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -75.76600646972656, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 367.02593994140625, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 19.728343963623047, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -1.338172435760498, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.9243640899658203, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 364.4987487792969, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -67.88621520996094, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 556.572265625, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01443049032241106, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.07016529142856598, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.04902814328670502, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.009884308092296124, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10600148886442184, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005886706057935953, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.07808923721313477, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.00032098640804179013, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7340236306190491, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59456.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.34512078762054443, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.6195271015167236, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.27629464864730835, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.18130914866924286, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1051027774810791, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.361125946044922, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.02394869551062584, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0741677284240723, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.07053180038928986, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.228529453277588, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72196.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8235130906105042, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -9.48674488067627, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 20.04774284362793, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 15.21958065032959, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.5053634643554688, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.0189061164855957, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 11.05747127532959, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.017353981733322144, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 19.70052719116211, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.029191438108682632, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11412616819143295, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1054249107837677, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.004929371643811464, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1677028089761734, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08481728285551071, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.7461204528808594, "metrics/particle/momentum/all/accepted/pz_bias": 0.0036012502387166023, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.154521942138672, "metrics/particle/momentum/all/accepted/samples": 249784.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0586574077606201, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1041576862335205, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6810605525970459, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5054172873497009, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.23558145761489868, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 4.413419246673584, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.2340562641620636, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.862228393554688, "metrics/particle/momentum/all/assigned/pz_bias": -0.08398311585187912, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.300264358520508, "metrics/particle/momentum/all/assigned/samples": 388146.0, "metrics/particle/momentum/all/efficiency": 0.6385724544525146, "metrics/particle/momentum/all/event/energy_bias": -54.612091064453125, "metrics/particle/momentum/all/event/p3_error_mean": 368.38555908203125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 26.659683227539062, "metrics/particle/momentum/all/event/px_bias": 0.7708714008331299, "metrics/particle/momentum/all/event/py_bias": -0.4128313660621643, "metrics/particle/momentum/all/event/pz_abs_mean": 364.5527038574219, "metrics/particle/momentum/all/event/pz_bias": -64.25102233886719, "metrics/particle/momentum/all/event/pz_residual_rms": 555.019287109375, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01600297912955284, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06258281320333481, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05218399688601494, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008858153596520424, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0973883867263794, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011782391229644418, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.20866811275482178, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0005558264092542231, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1520817279815674, "metrics/particle/momentum/class_1/accepted/samples": 172291.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.22415345907211304, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.9653078317642212, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.31275081634521484, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.25587305426597595, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07396207004785538, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.477241039276123, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.023170528933405876, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1131138801574707, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.017247619107365608, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.199149131774902, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8936811685562134, "metrics/particle/momentum/class_1/event/energy_bias": 44.32906723022461, "metrics/particle/momentum/class_1/event/p3_error_mean": 55.970123291015625, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 16.05886459350586, "metrics/particle/momentum/class_1/event/px_bias": 2.0391461849212646, "metrics/particle/momentum/class_1/event/py_bias": -0.9612671136856079, "metrics/particle/momentum/class_1/event/pz_abs_mean": 49.961692810058594, "metrics/particle/momentum/class_1/event/pz_bias": -1.0063576698303223, "metrics/particle/momentum/class_1/event/pz_residual_rms": 82.76937866210938, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.05842156335711479, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.24227112531661987, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23288094997406006, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07502992451190948, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27291086316108704, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.1132027804851532, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.240665435791016, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.03498053923249245, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 10.058423042297363, "metrics/particle/momentum/class_2/accepted/samples": 7915.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.2074530124664307, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.4793835878372192, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.131170392036438, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.64664626121521, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4718337059020996, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.393539905548096, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.3532387912273407, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.68938636779785, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.18714921176433563, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 34.88471221923828, "metrics/particle/momentum/class_2/assigned/samples": 31030.0, "metrics/particle/momentum/class_2/efficiency": 0.2547801434993744, "metrics/particle/momentum/class_2/event/energy_bias": -313.3003234863281, "metrics/particle/momentum/class_2/event/p3_error_mean": 168.81898498535156, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 28.728404998779297, "metrics/particle/momentum/class_2/event/px_bias": 1.125765323638916, "metrics/particle/momentum/class_2/event/py_bias": -4.192465305328369, "metrics/particle/momentum/class_2/event/pz_abs_mean": 162.60670471191406, "metrics/particle/momentum/class_2/event/pz_bias": -17.623624801635742, "metrics/particle/momentum/class_2/event/pz_residual_rms": 228.3152618408203, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.06066900119185448, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.22620455920696259, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.2227345108985901, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.010864106006920338, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.26237183809280396, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.31028029322624207, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.305244445800781, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.006198289804160595, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 13.975719451904297, "metrics/particle/momentum/class_3/accepted/samples": 21371.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 1.983416199684143, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.9394447207450867, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.046043038368225, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.8593164682388306, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.412734717130661, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 2.800462245941162, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5278047919273376, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.120697021484375, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.2551574110984802, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.438255310058594, "metrics/particle/momentum/class_3/assigned/samples": 52455.0, "metrics/particle/momentum/class_3/efficiency": 0.4009493291378021, "metrics/particle/momentum/class_3/event/energy_bias": 570.6380615234375, "metrics/particle/momentum/class_3/event/p3_error_mean": 277.3944091796875, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.587422370910645, "metrics/particle/momentum/class_3/event/px_bias": 0.20031331479549408, "metrics/particle/momentum/class_3/event/py_bias": -0.5289292335510254, "metrics/particle/momentum/class_3/event/pz_abs_mean": 276.9700927734375, "metrics/particle/momentum/class_3/event/pz_bias": -9.791786193847656, "metrics/particle/momentum/class_3/event/pz_residual_rms": 439.1144104003906, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.05993734300136566, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.23176075518131256, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22859923541545868, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.060681961476802826, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.267457515001297, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.35884034633636475, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 10.953822135925293, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.019245175644755363, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.331695556640625, "metrics/particle/momentum/class_4/accepted/samples": 36735.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.865440011024475, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0426288843154907, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0268217325210571, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.7416131496429443, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.36289283633232117, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.105742931365967, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5282612442970276, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.531970977783203, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.2347853183746338, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 40.18832778930664, "metrics/particle/momentum/class_4/assigned/samples": 82162.0, "metrics/particle/momentum/class_4/efficiency": 0.43584784865379333, "metrics/particle/momentum/class_4/event/energy_bias": -303.9610290527344, "metrics/particle/momentum/class_4/event/p3_error_mean": 322.61517333984375, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 11.02213191986084, "metrics/particle/momentum/class_4/event/px_bias": -1.395029902458191, "metrics/particle/momentum/class_4/event/py_bias": 1.2726733684539795, "metrics/particle/momentum/class_4/event/pz_abs_mean": 322.0594482421875, "metrics/particle/momentum/class_4/event/pz_bias": -38.451698303222656, "metrics/particle/momentum/class_4/event/pz_residual_rms": 478.9320373535156, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.05207238346338272, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22316309809684753, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21232248842716217, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.029348056763410568, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.25342968106269836, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02458484098315239, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.3621296882629395, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.002674335381016135, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.83098030090332, "metrics/particle/momentum/class_5/accepted/samples": 10779.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.440758466720581, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.108675003051758, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 1.0147050619125366, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.708719789981842, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.3803780674934387, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 8.079085350036621, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.14866779744625092, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.475882053375244, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.310079962015152, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 17.92432403564453, "metrics/particle/momentum/class_5/assigned/samples": 28937.0, "metrics/particle/momentum/class_5/efficiency": 0.372370183467865, "metrics/particle/momentum/class_5/event/energy_bias": -43.64302062988281, "metrics/particle/momentum/class_5/event/p3_error_mean": 113.42442321777344, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 25.06368064880371, "metrics/particle/momentum/class_5/event/px_bias": -0.4581318497657776, "metrics/particle/momentum/class_5/event/py_bias": 4.727452278137207, "metrics/particle/momentum/class_5/event/pz_abs_mean": 105.22936248779297, "metrics/particle/momentum/class_5/event/pz_bias": 5.25406551361084, "metrics/particle/momentum/class_5/event/pz_residual_rms": 165.47491455078125, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.01921446993947029, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.09036555141210556, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.0912550538778305, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.041979409754276276, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.1343754678964615, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.014000000432133675, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.723410964012146, "metrics/particle/momentum/class_6/accepted/pz_bias": -0.05554554983973503, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 2.605530023574829, "metrics/particle/momentum/class_6/accepted/samples": 500.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.3388441205024719, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 1.0234304666519165, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.6025166511535645, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.4520072042942047, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.10782608389854431, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 2.705592155456543, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.04695652052760124, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 2.2332346439361572, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.06866507977247238, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 9.405902862548828, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8695651888847351, "metrics/particle/momentum/class_6/event/energy_bias": -6.928119659423828, "metrics/particle/momentum/class_6/event/p3_error_mean": 8.574481010437012, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 3.05426287651062, "metrics/particle/momentum/class_6/event/px_bias": -0.7088509798049927, "metrics/particle/momentum/class_6/event/py_bias": -0.6861444115638733, "metrics/particle/momentum/class_6/event/pz_abs_mean": 7.609150409698486, "metrics/particle/momentum/class_6/event/pz_bias": -2.5132765769958496, "metrics/particle/momentum/class_6/event/pz_residual_rms": 27.059141159057617, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.0141295175999403, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.04277433827519417, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.03869563341140747, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.012328777462244034, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.07583367079496384, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.010362694039940834, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.6493048667907715, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.25213488936424255, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 2.7056260108947754, "metrics/particle/momentum/class_7/accepted/samples": 193.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.03939377889037132, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.09599403291940689, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.05636677145957947, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.025118818506598473, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.02512562833726406, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.14110228419303894, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.01005025114864111, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.6952595710754395, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.2166094332933426, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 2.709211587905884, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9698492288589478, "metrics/particle/momentum/class_7/event/energy_bias": -1.7466760873794556, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.2459464073181152, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 0.9485082626342773, "metrics/particle/momentum/class_7/event/px_bias": -0.03234075754880905, "metrics/particle/momentum/class_7/event/py_bias": -0.04415035992860794, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.8748183250427246, "metrics/particle/momentum/class_7/event/pz_bias": -0.11834578216075897, "metrics/particle/momentum/class_7/event/pz_residual_rms": 5.014849662780762, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 3.0717954635620117, "step_valid_time": 9772.173527240753, "step_total_time": 9775.245322942734, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0264693193981596, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3500348602889982, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 82214, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16088, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13158, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19745934335685794, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19568443330819568, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19656788177580656, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8043155666918043, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.16149739183798711, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.16004573430316005, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.16076828620127193, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19745934335685794, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0211790415015316, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.36094280067883777, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 82078, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15453, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12535, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19021652162138875, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18827213138721705, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18923933209647495, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.811727868612783, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15429781262694026, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15272058286020615, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15350514643301064, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19021652162138875, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0192796615614328, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.38328166044489786, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81558, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 14624, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11809, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.1810147421059798, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.1793079771450992, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18015731735110596, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8206920228549008, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1461708896013071, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1447926628902131, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.1454785120759854, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.1810147421059798} \ No newline at end of file diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_50000.json b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..12fa12759f330094cd540911faba000c97404efa --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 0.2154633755354265, "train": {"Classification_binary": 3.4272334575653076, "Classification": 0.015602258034050465, "Regression_pt": 0.4428579807281494, "Regression_eta": 0.06695611774921417, "Regression_sin_phi": 0.04634604603052139, "Regression_cos_phi": 0.04318489506840706, "Regression_energy": 0.48768508434295654, "Total": 14.202713012695312, "Total_unweighted": 4.529865839518607}, "train_group": {"physical_pu": {"Classification_binary": 2.9409526730631734, "Classification": 0.015512385917315772, "Regression_pt": 0.2287183790416508, "Regression_eta": 0.10501113362037219, "Regression_sin_phi": 0.04724516924265977, "Regression_cos_phi": 0.04722035594366409, "Regression_energy": 0.27537913964344907, "Total": 11.810755603916043, "Total_unweighted": 3.660039236472285}, "physical_nopu": {"Classification_binary": 3.4963054914023353, "Classification": 0.014402996026372948, "Regression_pt": 0.3742879396027858, "Regression_eta": 0.06908588035993188, "Regression_sin_phi": 0.04810049772115037, "Regression_cos_phi": 0.048069258367005485, "Regression_energy": 0.4207635274027834, "Total": 13.670528952175765, "Total_unweighted": 4.471015590882366}}, "valid": {"Classification_binary": 3.21232541402181, "Classification": 0.014971869687239328, "Regression_pt": 0.3017948468526204, "Regression_eta": 0.08804288506507874, "Regression_sin_phi": 0.047898401816686, "Regression_cos_phi": 0.04776974022388458, "Regression_energy": 0.34777724742889404, "Total": 12.76425552368164, "group/physical_pu/Classification": 0.015543051064014435, "group/physical_pu/Classification_binary": 2.927492141723633, "group/physical_pu/Regression_cos_phi": 0.04741188883781433, "group/physical_pu/Regression_energy": 0.2758038441340129, "group/physical_pu/Regression_eta": 0.10491541028022766, "group/physical_pu/Regression_pt": 0.2292285362879435, "group/physical_pu/Regression_sin_phi": 0.04735828936100006, "group/physical_pu/Total": 11.814425150553385, "group/physical_pu/Total_unweighted": 3.647753161688646, "group/physical_nopu/Classification": 0.014400688310464224, "group/physical_nopu/Classification_binary": 3.497159004211426, "group/physical_nopu/Regression_cos_phi": 0.0481275866429011, "group/physical_nopu/Regression_energy": 0.4197506904602051, "group/physical_nopu/Regression_eta": 0.07117036481698354, "group/physical_nopu/Regression_pt": 0.3743611176808675, "group/physical_nopu/Regression_sin_phi": 0.0484385093053182, "group/physical_nopu/Total": 13.71408208211263, "group/physical_nopu/Total_unweighted": 4.473407961428165, "Total_unweighted": 4.060580405096213, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.014790177345275879, "diagnostic/loss/cms_pf/Classification_binary": 3.2075281143188477, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.04737263545393944, "diagnostic/loss/cms_pf/Regression_energy": 0.3354518711566925, "diagnostic/loss/cms_pf/Regression_eta": 0.08808627724647522, "diagnostic/loss/cms_pf/Regression_pt": 0.2895745635032654, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.04621943086385727, "diagnostic/regression/cms_pf/energy_pred_mean": 0.20093700289726257, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.35416585206985474, "diagnostic/regression/cms_pf/energy_residual_mean": 0.09017077088356018, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5850862264633179, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.11331628262996674, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.32659614086151123, "diagnostic/regression/cms_pf/pt_residual_mean": 0.12408845126628876, "diagnostic/regression/cms_pf/pt_residual_rms": 0.545070469379425, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.896581768989563, "metrics/particle/class_1/pid_accuracy": 0.9786962270736694, "metrics/particle/class_2/efficiency": 0.2574116885662079, "metrics/particle/class_2/pid_accuracy": 0.20798900723457336, "metrics/particle/class_3/efficiency": 0.3953016698360443, "metrics/particle/class_3/pid_accuracy": 0.4823204576969147, "metrics/particle/class_4/efficiency": 0.441263884305954, "metrics/particle/class_4/pid_accuracy": 0.6564093828201294, "metrics/particle/class_5/efficiency": 0.3669886887073517, "metrics/particle/class_5/pid_accuracy": 0.6236711740493774, "metrics/particle/class_6/efficiency": 0.8680742383003235, "metrics/particle/class_6/pid_accuracy": 0.059908587485551834, "metrics/particle/class_7/efficiency": 0.9646545052528381, "metrics/particle/class_7/pid_accuracy": 0.2461918443441391, "metrics/particle/count/bias_mean": 73.36849212646484, "metrics/particle/count/mae": 95.306640625, "metrics/particle/count/prediction_mean": 1601.9720458984375, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.07859079539775848, "metrics/particle/event/energy_response_mean": 0.9552281498908997, "metrics/particle/event/met_abs_error": 21.365564346313477, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05087621137499809, "metrics/particle/event/scalar_pt_response_mean": 1.0136855840682983, "metrics/particle/event/vector_pt_closure": 0.02918076328933239, "metrics/particle/matched/delta_eta_abs_mean": 0.018997400999069214, "metrics/particle/matched/delta_phi_abs_mean": 0.0165219958871603, "metrics/particle/matched/delta_r_mean": 0.028186550363898277, "metrics/particle/matched/energy_relative_abs_mean": 0.11431104689836502, "metrics/particle/matched/pid_accuracy": 0.8466396927833557, "metrics/particle/matched/pt_relative_abs_mean": 0.10496189445257187, "metrics/particle/matching/duplicate_fraction": 0.1102057620882988, "metrics/particle/matching/efficiency": 0.6399549245834351, "metrics/particle/matching/f1": 0.6249568462371826, "metrics/particle/matching/purity": 0.6106457114219666, "metrics/particle/matching/target_coverage_dr0p05": 0.516320526599884, "metrics/particle/matching/target_coverage_dr0p10": 0.703355073928833, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5590001344680786, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7575405836105347, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017937228083610535, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06675230711698532, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.058216262608766556, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0017671632813289762, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.11057450622320175, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0017709392122924328, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.2146565318107605, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.01125295553356409, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.3937448263168335, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68890.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.46454110741615295, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.086252212524414, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.36874258518218994, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.2412431687116623, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13261067867279053, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.158442497253418, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03949109837412834, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.780874490737915, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.08905891329050064, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 8.656105041503906, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87640.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.785957932472229, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 40.78520202636719, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 45.980628967285156, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 17.69456672668457, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.6990199089050293, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.5012943148612976, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 40.170772552490234, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 6.13419246673584, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 72.59864807128906, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04066087678074837, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16050727665424347, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15806981921195984, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.014385811053216457, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21160024404525757, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.1729723960161209, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.475478649139404, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.008229727856814861, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.758559226989746, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 121609.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5195239782333374, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.949455976486206, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9510756134986877, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7374471426010132, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3183872103691101, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3.8252573013305664, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.3765139579772949, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.684223175048828, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.09580396115779877, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.07455062866211, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228621.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5257380604743958, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 398.41265869140625, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 377.117919921875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 19.927227020263672, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 1.5640703439712524, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.6566109657287598, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 375.1869201660156, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -84.46034240722656, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 563.1185913085938, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014138945378363132, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.07343876361846924, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.04931584373116493, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.006255372427403927, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1067768856883049, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007385400240309536, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.07891640067100525, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.002104489365592599, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.736667275428772, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59577.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.34345245361328125, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3627969026565552, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.25931158661842346, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.16062425076961517, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10478854924440384, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 1.9427056312561035, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.023700358346104622, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.138150691986084, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.04762697219848633, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.954599380493164, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72193.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8251890540122986, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.5418286323547363, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 22.766998291015625, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 17.04439353942871, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 4.885640621185303, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.7945066690444946, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 12.890174865722656, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.6581866145133972, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 30.162778854370117, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.028082579374313354, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11393719166517258, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1046534851193428, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.005992209538817406, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1669064313173294, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08477822691202164, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.740593910217285, "metrics/particle/momentum/all/accepted/pz_bias": 0.007603303994983435, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.2401704788208, "metrics/particle/momentum/all/accepted/samples": 250076.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0629374980926514, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.057137131690979, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6911318898200989, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5182965993881226, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.23677706718444824, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.2193760871887207, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.2349081188440323, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.84411334991455, "metrics/particle/momentum/all/assigned/pz_bias": -0.08532863855361938, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.127086639404297, "metrics/particle/momentum/all/assigned/samples": 388454.0, "metrics/particle/momentum/all/efficiency": 0.6393189430236816, "metrics/particle/momentum/all/event/energy_bias": 437.6559753417969, "metrics/particle/momentum/all/event/p3_error_mean": 376.1498718261719, "metrics/particle/momentum/all/event/pt_vector_error_mean": 28.80132293701172, "metrics/particle/momentum/all/event/px_bias": 8.148731231689453, "metrics/particle/momentum/all/event/py_bias": -1.9498236179351807, "metrics/particle/momentum/all/event/pz_abs_mean": 371.7364196777344, "metrics/particle/momentum/all/event/pz_bias": -78.98432922363281, "metrics/particle/momentum/all/event/pz_residual_rms": 560.9527587890625, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.014551958069205284, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06292613595724106, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.051598772406578064, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004718707874417305, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09653796255588531, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011976532405242324, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1999874860048294, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0075536081567406654, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.0725977420806885, "metrics/particle/momentum/class_1/accepted/samples": 172838.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.21871896088123322, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.8485230803489685, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.30172815918922424, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24151542782783508, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07094321399927139, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.696288824081421, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.022506587207317352, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1151734590530396, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.014800992794334888, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.31443452835083, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8965184688568115, "metrics/particle/momentum/class_1/event/energy_bias": 28.864307403564453, "metrics/particle/momentum/class_1/event/p3_error_mean": 56.2779541015625, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.265085220336914, "metrics/particle/momentum/class_1/event/px_bias": 1.8920459747314453, "metrics/particle/momentum/class_1/event/py_bias": -2.6677029132843018, "metrics/particle/momentum/class_1/event/pz_abs_mean": 50.9939079284668, "metrics/particle/momentum/class_1/event/pz_bias": 11.655312538146973, "metrics/particle/momentum/class_1/event/pz_residual_rms": 85.62307739257812, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.057958025485277176, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23927728831768036, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23092292249202728, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07018206268548965, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2709050476551056, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.11128029227256775, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.232859134674072, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.14282436668872833, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 10.583754539489746, "metrics/particle/momentum/class_2/accepted/samples": 7881.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.1902031898498535, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.5802778005599976, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.156803846359253, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6684494614601135, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4715868830680847, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.486998558044434, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.35455191135406494, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.759737014770508, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.007813853211700916, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 35.16417694091797, "metrics/particle/momentum/class_2/assigned/samples": 31042.0, "metrics/particle/momentum/class_2/efficiency": 0.25368571281433105, "metrics/particle/momentum/class_2/event/energy_bias": -278.0155334472656, "metrics/particle/momentum/class_2/event/p3_error_mean": 167.5589141845703, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 28.371322631835938, "metrics/particle/momentum/class_2/event/px_bias": 3.251650333404541, "metrics/particle/momentum/class_2/event/py_bias": -1.4922329187393188, "metrics/particle/momentum/class_2/event/pz_abs_mean": 161.9781494140625, "metrics/particle/momentum/class_2/event/pz_bias": 3.2200393676757812, "metrics/particle/momentum/class_2/event/pz_residual_rms": 225.57777404785156, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.06074102222919464, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.22607627511024475, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.2228834331035614, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.005925042554736137, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2624700367450714, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.31194573640823364, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.333120346069336, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.05932221934199333, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 13.955546379089355, "metrics/particle/momentum/class_3/accepted/samples": 21087.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.0462770462036133, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.944273829460144, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.0978150367736816, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.921390950679779, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4219309687614441, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 3.269486665725708, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5355673432350159, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.34125518798828, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.15135249495506287, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.52545928955078, "metrics/particle/momentum/class_3/assigned/samples": 52492.0, "metrics/particle/momentum/class_3/efficiency": 0.39562109112739563, "metrics/particle/momentum/class_3/event/energy_bias": 703.8182373046875, "metrics/particle/momentum/class_3/event/p3_error_mean": 282.55084228515625, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.601568222045898, "metrics/particle/momentum/class_3/event/px_bias": 0.7159554958343506, "metrics/particle/momentum/class_3/event/py_bias": -0.22711089253425598, "metrics/particle/momentum/class_3/event/pz_abs_mean": 282.1365966796875, "metrics/particle/momentum/class_3/event/pz_bias": -10.655523300170898, "metrics/particle/momentum/class_3/event/pz_residual_rms": 444.9147033691406, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.05976787954568863, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.23079417645931244, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22767482697963715, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.05360320210456848, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.26664817333221436, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.35925453901290894, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 10.989117622375488, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.019683031365275383, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.62977409362793, "metrics/particle/momentum/class_4/accepted/samples": 36971.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.8547704219818115, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.044851541519165, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0713036060333252, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8018499612808228, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3682435154914856, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.347574234008789, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5265405774116516, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.213777542114258, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.2751210927963257, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 39.48789596557617, "metrics/particle/momentum/class_4/assigned/samples": 82421.0, "metrics/particle/momentum/class_4/efficiency": 0.43864789605140686, "metrics/particle/momentum/class_4/event/energy_bias": 5.84644889831543, "metrics/particle/momentum/class_4/event/p3_error_mean": 325.8669128417969, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 11.009969711303711, "metrics/particle/momentum/class_4/event/px_bias": -0.00040262192487716675, "metrics/particle/momentum/class_4/event/py_bias": -0.7603657245635986, "metrics/particle/momentum/class_4/event/pz_abs_mean": 325.2333068847656, "metrics/particle/momentum/class_4/event/pz_bias": -85.79906463623047, "metrics/particle/momentum/class_4/event/pz_residual_rms": 489.1348876953125, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.05174342542886734, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22419586777687073, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2133491337299347, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.034202661365270615, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2542630136013031, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.023470638319849968, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.3109217882156372, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.002216530032455921, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.276536464691162, "metrics/particle/momentum/class_5/accepted/samples": 10609.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4602304697036743, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.1344926357269287, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9721750617027283, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6677412986755371, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.3835228383541107, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.199804782867432, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.1509140580892563, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.525704383850098, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.19301050901412964, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 17.638734817504883, "metrics/particle/momentum/class_5/assigned/samples": 28937.0, "metrics/particle/momentum/class_5/efficiency": 0.3664973974227905, "metrics/particle/momentum/class_5/event/energy_bias": -13.841798782348633, "metrics/particle/momentum/class_5/event/p3_error_mean": 116.64899444580078, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 26.131908416748047, "metrics/particle/momentum/class_5/event/px_bias": 3.077085494995117, "metrics/particle/momentum/class_5/event/py_bias": 3.8746981620788574, "metrics/particle/momentum/class_5/event/pz_abs_mean": 108.36088562011719, "metrics/particle/momentum/class_5/event/pz_bias": 4.924061298370361, "metrics/particle/momentum/class_5/event/pz_residual_rms": 171.13392639160156, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.01775437220931053, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.08955360949039459, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.0899108424782753, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.0386105552315712, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.1335851103067398, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.01411290280520916, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.6712608933448792, "metrics/particle/momentum/class_6/accepted/pz_bias": 0.06702937185764313, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 3.0953261852264404, "metrics/particle/momentum/class_6/accepted/samples": 496.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.33595985174179077, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 0.9299789667129517, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.5660863518714905, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.41875195503234863, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.10608695447444916, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 2.7639310359954834, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.04521739110350609, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 1.9892714023590088, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.06646140664815903, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 8.217618942260742, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8626086711883545, "metrics/particle/momentum/class_6/event/energy_bias": -7.58880615234375, "metrics/particle/momentum/class_6/event/p3_error_mean": 8.506072998046875, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 3.0746495723724365, "metrics/particle/momentum/class_6/event/px_bias": -0.6489325165748596, "metrics/particle/momentum/class_6/event/py_bias": -0.5761038661003113, "metrics/particle/momentum/class_6/event/pz_abs_mean": 7.516871452331543, "metrics/particle/momentum/class_6/event/pz_bias": -2.1715285778045654, "metrics/particle/momentum/class_6/event/pz_residual_rms": 26.925148010253906, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.013426324352622032, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.04282334819436073, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.040543075650930405, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.010440601967275143, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.0817951038479805, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.005154639016836882, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.543977677822113, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.08376636356115341, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 1.940070629119873, "metrics/particle/momentum/class_7/accepted/samples": 194.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.07324949651956558, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.1206321269273758, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.06082290783524513, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.02504870668053627, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.015075377188622952, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.212319016456604, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.01005025114864111, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.6248268485069275, "metrics/particle/momentum/class_7/assigned/pz_bias": 0.008267596364021301, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 2.0888190269470215, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9748743772506714, "metrics/particle/momentum/class_7/event/energy_bias": -1.4268271923065186, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.0912375450134277, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 0.9292017221450806, "metrics/particle/momentum/class_7/event/px_bias": -0.13867099583148956, "metrics/particle/momentum/class_7/event/py_bias": -0.10100562870502472, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.7175174951553345, "metrics/particle/momentum/class_7/event/pz_bias": -0.15763168036937714, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.600162506103516, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 3.6919586658477783, "step_valid_time": 12265.729289531708, "step_total_time": 12269.421248435974, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0321313680874895, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3459335268651962, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81519, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15767, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12915, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19351948450444922, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19341503207841118, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.1934672441930378, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8065849679215888, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15851488186560295, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15842932322526038, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15847209099721463, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19351948450444922, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0299721449315207, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.36808267842205766, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81320, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15367, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12454, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18915791676411575, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18896950319724545, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18906366303926575, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8110304968027545, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15330075456369477, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15314805705853418, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15322436776801038, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18915791676411575, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0285990625007733, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3981446788716564, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 80763, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 14517, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11612, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.1796903043731201, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.17974815200029717, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.1797192235317421, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8202518479997029, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14373243882211686, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14377871054814706, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.1437555709616718, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.1796903043731201} \ No newline at end of file diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_60000.json b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_60000.json new file mode 100644 index 0000000000000000000000000000000000000000..9261fc876c4bddc601a17c23f0851643b380c7a5 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_60000.json @@ -0,0 +1 @@ +{"epoch": 0.2585560506425118, "train": {"Classification_binary": 3.3905110359191895, "Classification": 0.015391472727060318, "Regression_pt": 0.5450783967971802, "Regression_eta": 0.0703045055270195, "Regression_sin_phi": 0.04308547452092171, "Regression_cos_phi": 0.04560207203030586, "Regression_energy": 0.5805318355560303, "Total": 15.399974822998047, "Total_unweighted": 4.690504793077707}, "train_group": {"physical_pu": {"Classification_binary": 2.9225644934308397, "Classification": 0.015569235566635054, "Regression_pt": 0.22819161120351855, "Regression_eta": 0.1050401468362127, "Regression_sin_phi": 0.047248020399730285, "Regression_cos_phi": 0.04723192530346441, "Regression_energy": 0.2747155326408344, "Total": 11.787784964173705, "Total_unweighted": 3.640560965381235}, "physical_nopu": {"Classification_binary": 3.4821347531968088, "Classification": 0.014329581946642748, "Regression_pt": 0.37159944059300604, "Regression_eta": 0.06977033896250824, "Regression_sin_phi": 0.047907596808521136, "Regression_cos_phi": 0.04813297756827704, "Regression_energy": 0.41844647729357476, "Total": 13.627501719593345, "Total_unweighted": 4.452321166369338}}, "valid": {"Classification_binary": 3.2011372248331704, "Classification": 0.014938621471325556, "Regression_pt": 0.29933764537175495, "Regression_eta": 0.08783543109893799, "Regression_sin_phi": 0.0478644867738088, "Regression_cos_phi": 0.047695900003115334, "Regression_energy": 0.3452478249867757, "Total": 12.713569641113281, "group/physical_pu/Classification": 0.015514961133400599, "group/physical_pu/Classification_binary": 2.919404983520508, "group/physical_pu/Regression_cos_phi": 0.04737235605716705, "group/physical_pu/Regression_energy": 0.27500659227371216, "group/physical_pu/Regression_eta": 0.1047972043355306, "group/physical_pu/Regression_pt": 0.22827581564585367, "group/physical_pu/Regression_sin_phi": 0.0473142663637797, "group/physical_pu/Total": 11.789311726888021, "group/physical_pu/Total_unweighted": 3.6376861793299518, "group/physical_nopu/Classification": 0.014362281809250513, "group/physical_nopu/Classification_binary": 3.4828691482543945, "group/physical_nopu/Regression_cos_phi": 0.04801945388317108, "group/physical_nopu/Regression_energy": 0.41548903783162433, "group/physical_nopu/Regression_eta": 0.07087367276350658, "group/physical_nopu/Regression_pt": 0.3703994353612264, "group/physical_nopu/Regression_sin_phi": 0.048414712150891624, "group/physical_nopu/Total": 13.637825012207031, "group/physical_nopu/Total_unweighted": 4.450427742054065, "Total_unweighted": 4.044057134538889, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.014764530584216118, "diagnostic/loss/cms_pf/Classification_binary": 3.198697090148926, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.04738575592637062, "diagnostic/loss/cms_pf/Regression_energy": 0.33321666717529297, "diagnostic/loss/cms_pf/Regression_eta": 0.08796640485525131, "diagnostic/loss/cms_pf/Regression_pt": 0.28769731521606445, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.04610658437013626, "diagnostic/regression/cms_pf/energy_pred_mean": 0.2058815062046051, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.35315680503845215, "diagnostic/regression/cms_pf/energy_residual_mean": 0.0951152890920639, "diagnostic/regression/cms_pf/energy_residual_rms": 0.584894597530365, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.11376124620437622, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.32533425092697144, "diagnostic/regression/cms_pf/pt_residual_mean": 0.12453342229127884, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5439143776893616, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8957262635231018, "metrics/particle/class_1/pid_accuracy": 0.9785038828849792, "metrics/particle/class_2/efficiency": 0.25658920407295227, "metrics/particle/class_2/pid_accuracy": 0.2068285048007965, "metrics/particle/class_3/efficiency": 0.3940688371658325, "metrics/particle/class_3/pid_accuracy": 0.4785548448562622, "metrics/particle/class_4/efficiency": 0.43831291794776917, "metrics/particle/class_4/pid_accuracy": 0.6577773690223694, "metrics/particle/class_5/efficiency": 0.37254598736763, "metrics/particle/class_5/pid_accuracy": 0.6323926448822021, "metrics/particle/class_6/efficiency": 0.8602805733680725, "metrics/particle/class_6/pid_accuracy": 0.06308680772781372, "metrics/particle/class_7/efficiency": 0.9682287573814392, "metrics/particle/class_7/pid_accuracy": 0.2530762851238251, "metrics/particle/count/bias_mean": 67.24283599853516, "metrics/particle/count/mae": 87.77669525146484, "metrics/particle/count/prediction_mean": 1595.8463134765625, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.084773488342762, "metrics/particle/event/energy_response_mean": 0.9528091549873352, "metrics/particle/event/met_abs_error": 19.361886978149414, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04738282039761543, "metrics/particle/event/scalar_pt_response_mean": 1.0097781419754028, "metrics/particle/event/vector_pt_closure": 0.027981946244835854, "metrics/particle/matched/delta_eta_abs_mean": 0.020399583503603935, "metrics/particle/matched/delta_phi_abs_mean": 0.015956206247210503, "metrics/particle/matched/delta_r_mean": 0.029121901839971542, "metrics/particle/matched/energy_relative_abs_mean": 0.11349371820688248, "metrics/particle/matched/pid_accuracy": 0.8468733429908752, "metrics/particle/matched/pt_relative_abs_mean": 0.1043262928724289, "metrics/particle/matching/duplicate_fraction": 0.10891801118850708, "metrics/particle/matching/efficiency": 0.6390541195869446, "metrics/particle/matching/f1": 0.6253007054328918, "metrics/particle/matching/purity": 0.6121268272399902, "metrics/particle/matching/target_coverage_dr0p05": 0.5129445791244507, "metrics/particle/matching/target_coverage_dr0p10": 0.7015049457550049, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5599575638771057, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7575831413269043, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01939385198056698, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0665665790438652, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05811964347958565, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.00048032915219664574, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.11056700348854065, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0019738178234547377, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.22024627029895782, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0015420522540807724, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.3420913219451904, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68902.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.47156476974487305, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1625181436538696, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.37102043628692627, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.24674905836582184, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13306939601898193, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.1322731971740723, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.0408233106136322, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.88327157497406, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.11475467681884766, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.808557510375977, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87646.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7860948443412781, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 29.930984497070312, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 41.72612762451172, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 16.2510986328125, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.579763650894165, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.8749181032180786, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 36.50785446166992, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.6547201871871948, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 56.21309280395508, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04210611432790756, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16060203313827515, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1576230227947235, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.01406586542725563, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21125292778015137, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17397496104240417, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.485936641693115, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.04619108512997627, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.567928314208984, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 121483.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5196107625961304, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9887094497680664, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9492437243461609, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7369941473007202, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3190865218639374, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3.539172410964966, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.3775556981563568, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.751253128051758, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.12954990565776825, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.19684982299805, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228753.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5251933336257935, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 537.8505249023438, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 387.46539306640625, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 19.897085189819336, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 1.1676256656646729, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 1.593123435974121, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 385.9051818847656, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -118.37770080566406, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 579.3189086914062, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013572603464126587, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0720762312412262, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.04916013404726982, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008805972523987293, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10685926675796509, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007528734859079123, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.07891597598791122, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0057447561994194984, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7912195920944214, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59771.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.33697545528411865, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3060595989227295, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.2522878646850586, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.15700340270996094, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10360541194677353, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 2.228802442550659, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.022590966895222664, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0888943672180176, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0639224573969841, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.709092140197754, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72197.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.827876091003418, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -9.966596603393555, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 21.06214141845703, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 15.623446464538574, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.5463372468948364, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.9408906698226929, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 11.953712463378906, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.7716896533966064, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 22.576387405395508, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.029032684862613678, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1135493665933609, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10430063307285309, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.004594438709318638, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16663777828216553, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08521082997322083, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.743649482727051, "metrics/particle/momentum/all/accepted/pz_bias": -0.02422908879816532, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.101302146911621, "metrics/particle/momentum/all/accepted/samples": 250156.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0635079145431519, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0868713855743408, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6893411874771118, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.518665611743927, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2370971441268921, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.053170680999756, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.23565863072872162, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.899285316467285, "metrics/particle/momentum/all/assigned/pz_bias": -0.11402002722024918, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.30403709411621, "metrics/particle/momentum/all/assigned/samples": 388596.0, "metrics/particle/momentum/all/efficiency": 0.639523446559906, "metrics/particle/momentum/all/event/energy_bias": 557.81494140625, "metrics/particle/momentum/all/event/p3_error_mean": 387.0599365234375, "metrics/particle/momentum/all/event/pt_vector_error_mean": 26.025779724121094, "metrics/particle/momentum/all/event/px_bias": 4.293726444244385, "metrics/particle/momentum/all/event/py_bias": 2.527150869369507, "metrics/particle/momentum/all/event/pz_abs_mean": 383.77032470703125, "metrics/particle/momentum/all/event/pz_bias": -120.8041000366211, "metrics/particle/momentum/all/event/pz_residual_rms": 579.30615234375, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.015830151736736298, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06216130405664444, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.051073841750621796, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006490956526249647, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09569951146841049, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012617274187505245, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.20278829336166382, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.00826698075979948, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1177234649658203, "metrics/particle/momentum/class_1/accepted/samples": 172779.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.2247808277606964, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.8943084478378296, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.309859037399292, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2524486184120178, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07188206911087036, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.619075059890747, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.02316015586256981, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1653194427490234, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.012253822758793831, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.790295600891113, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8962123990058899, "metrics/particle/momentum/class_1/event/energy_bias": 28.251224517822266, "metrics/particle/momentum/class_1/event/p3_error_mean": 55.424198150634766, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.628276824951172, "metrics/particle/momentum/class_1/event/px_bias": 0.6689213514328003, "metrics/particle/momentum/class_1/event/py_bias": -0.5630508065223694, "metrics/particle/momentum/class_1/event/pz_abs_mean": 50.460365295410156, "metrics/particle/momentum/class_1/event/pz_bias": -14.191230773925781, "metrics/particle/momentum/class_1/event/pz_residual_rms": 83.6913070678711, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.058196280151605606, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.24090784788131714, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23151211440563202, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06791427731513977, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27178382873535156, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.11010785400867462, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.162026882171631, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.06807690858840942, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 9.892437934875488, "metrics/particle/momentum/class_2/accepted/samples": 7974.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.1963300704956055, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.5446871519088745, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1239227056503296, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6398230195045471, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4718080759048462, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.958739995956421, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.35392045974731445, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.68254852294922, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.25673308968544006, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 34.83784866333008, "metrics/particle/momentum/class_2/assigned/samples": 31055.0, "metrics/particle/momentum/class_2/efficiency": 0.25667932629585266, "metrics/particle/momentum/class_2/event/energy_bias": -227.24363708496094, "metrics/particle/momentum/class_2/event/p3_error_mean": 165.79598999023438, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 28.037641525268555, "metrics/particle/momentum/class_2/event/px_bias": 2.125244379043579, "metrics/particle/momentum/class_2/event/py_bias": -0.8326917886734009, "metrics/particle/momentum/class_2/event/pz_abs_mean": 159.66273498535156, "metrics/particle/momentum/class_2/event/pz_bias": -43.2519416809082, "metrics/particle/momentum/class_2/event/pz_residual_rms": 224.50587463378906, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.060803063213825226, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.22684703767299652, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.22310762107372284, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.005372033454477787, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2628505825996399, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.3170013725757599, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.444890975952148, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.10882191359996796, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 14.135557174682617, "metrics/particle/momentum/class_3/accepted/samples": 21063.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.0233078002929688, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.9745674133300781, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.0888043642044067, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.9140626192092896, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4203997552394867, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 3.1041202545166016, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5355843305587769, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.39433479309082, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.14626376330852509, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.47300338745117, "metrics/particle/momentum/class_3/assigned/samples": 52481.0, "metrics/particle/momentum/class_3/efficiency": 0.39517080783843994, "metrics/particle/momentum/class_3/event/energy_bias": 693.1547241210938, "metrics/particle/momentum/class_3/event/p3_error_mean": 281.2610778808594, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.439393997192383, "metrics/particle/momentum/class_3/event/px_bias": 0.17978504300117493, "metrics/particle/momentum/class_3/event/py_bias": 0.29032278060913086, "metrics/particle/momentum/class_3/event/pz_abs_mean": 280.81488037109375, "metrics/particle/momentum/class_3/event/pz_bias": 10.544453620910645, "metrics/particle/momentum/class_3/event/pz_residual_rms": 446.7201232910156, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.06013533100485802, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.2300024926662445, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22689805924892426, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.05458784103393555, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.26609715819358826, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.36061742901802063, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 10.987578392028809, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.08103030174970627, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.18769645690918, "metrics/particle/momentum/class_4/accepted/samples": 36798.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.8603914976119995, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0801291465759277, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0642774105072021, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.795600175857544, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.369172066450119, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.1772139072418213, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5299860835075378, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.319793701171875, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.2668687105178833, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 39.77570343017578, "metrics/particle/momentum/class_4/assigned/samples": 82555.0, "metrics/particle/momentum/class_4/efficiency": 0.4365953207015991, "metrics/particle/momentum/class_4/event/energy_bias": 98.6612777709961, "metrics/particle/momentum/class_4/event/p3_error_mean": 322.4468078613281, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 10.879171371459961, "metrics/particle/momentum/class_4/event/px_bias": 0.4790835678577423, "metrics/particle/momentum/class_4/event/py_bias": 0.8972127437591553, "metrics/particle/momentum/class_4/event/pz_abs_mean": 321.7500305175781, "metrics/particle/momentum/class_4/event/pz_bias": -67.60665130615234, "metrics/particle/momentum/class_4/event/pz_residual_rms": 484.62353515625, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.051467955112457275, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22570861876010895, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21374069154262543, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.037744730710983276, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2546701431274414, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.024316109716892242, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.321694254875183, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.012304779142141342, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.534598350524902, "metrics/particle/momentum/class_5/accepted/samples": 10857.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4418940544128418, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.1086442470550537, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9636684656143188, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6603577733039856, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.3806101679801941, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.026520252227783, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.14625297486782074, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.556631088256836, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.13613128662109375, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 18.548620223999023, "metrics/particle/momentum/class_5/assigned/samples": 28943.0, "metrics/particle/momentum/class_5/efficiency": 0.37506476044654846, "metrics/particle/momentum/class_5/event/energy_bias": -25.695552825927734, "metrics/particle/momentum/class_5/event/p3_error_mean": 110.69775390625, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 24.77362632751465, "metrics/particle/momentum/class_5/event/px_bias": 1.4013993740081787, "metrics/particle/momentum/class_5/event/py_bias": 3.373755931854248, "metrics/particle/momentum/class_5/event/pz_abs_mean": 102.53038024902344, "metrics/particle/momentum/class_5/event/pz_bias": -5.10651159286499, "metrics/particle/momentum/class_5/event/pz_residual_rms": 160.24021911621094, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.017597028985619545, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.08991818875074387, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.09027164429426193, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.03804963454604149, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.1354096382856369, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.012195121496915817, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.8090360760688782, "metrics/particle/momentum/class_6/accepted/pz_bias": 0.012696750462055206, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 4.25726842880249, "metrics/particle/momentum/class_6/accepted/samples": 492.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.38322731852531433, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 0.9180365800857544, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.5648234486579895, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.4164062440395355, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.11130435019731522, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 2.946014165878296, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.038260869681835175, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 2.139648675918579, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.4340131878852844, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 9.52286148071289, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8556521534919739, "metrics/particle/momentum/class_6/event/energy_bias": -7.864274978637695, "metrics/particle/momentum/class_6/event/p3_error_mean": 6.73409366607666, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 2.494276285171509, "metrics/particle/momentum/class_6/event/px_bias": -0.41024112701416016, "metrics/particle/momentum/class_6/event/py_bias": -0.5411342978477478, "metrics/particle/momentum/class_6/event/pz_abs_mean": 5.863649368286133, "metrics/particle/momentum/class_6/event/pz_bias": -1.067215919494629, "metrics/particle/momentum/class_6/event/pz_residual_rms": 17.069433212280273, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.013084550388157368, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.038438402116298676, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.03707687184214592, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.007883683778345585, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.07102672010660172, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.015544041059911251, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.562021017074585, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.2150643914937973, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 2.160125255584717, "metrics/particle/momentum/class_7/accepted/samples": 193.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.04984653741121292, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.48751094937324524, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.07804474234580994, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.04393773525953293, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.02512562833726406, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.3334898352622986, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.02010050229728222, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.8121131658554077, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.37924158573150635, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 3.425022840499878, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9698492288589478, "metrics/particle/momentum/class_7/event/energy_bias": -1.448873519897461, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.237408399581909, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 0.9854123592376709, "metrics/particle/momentum/class_7/event/px_bias": -0.15046627819538116, "metrics/particle/momentum/class_7/event/py_bias": -0.09726351499557495, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.83658766746521, "metrics/particle/momentum/class_7/event/pz_bias": -0.1250135749578476, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.861629486083984, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 3.208068370819092, "step_valid_time": 14711.008471012115, "step_total_time": 14714.216539859772, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.02757611608251, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3416318035662873, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81656, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 16090, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13225, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19748389076403805, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19704614480258645, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.1972647749354813, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8029538551974136, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.162319729978521, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1619599294601744, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.16213963011322188, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19748389076403805, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0214770643699473, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.35135484609149503, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81628, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15422, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12571, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18983493149841824, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18893026902533444, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18938151989046276, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8110697309746655, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15474094954393824, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15400352820110746, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.1543713582248092, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18983493149841824, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.020500475880157, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.37880664919977025, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81011, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 14693, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11857, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.1818688187748332, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18137043117601315, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18161928306551298, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8186295688239869, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14676502989268342, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14636283961437335, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14656365883807168, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.1818688187748332} \ No newline at end of file diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_70000.json b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_70000.json new file mode 100644 index 0000000000000000000000000000000000000000..db2825c856e64e5d0bfb877f7af1129d556fdbbd --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_70000.json @@ -0,0 +1 @@ +{"epoch": 0.30164872574959706, "train": {"Classification_binary": 3.3872971534729004, "Classification": 0.01625281572341919, "Regression_pt": 0.5683447122573853, "Regression_eta": 0.06289155781269073, "Regression_sin_phi": 0.0456213653087616, "Regression_cos_phi": 0.04979468509554863, "Regression_energy": 0.6014564037322998, "Total": 15.847684860229492, "Total_unweighted": 4.731658693403006}, "train_group": {"physical_pu": {"Classification_binary": 2.914890147827484, "Classification": 0.01557972199273306, "Regression_pt": 0.22780440338365324, "Regression_eta": 0.10469297434274967, "Regression_sin_phi": 0.047165158276374526, "Regression_cos_phi": 0.047094345829644044, "Regression_energy": 0.27423536286249267, "Total": 11.761292499500316, "Total_unweighted": 3.631462114515131}, "physical_nopu": {"Classification_binary": 3.4707600682219786, "Classification": 0.01428507943714511, "Regression_pt": 0.36594746077414786, "Regression_eta": 0.06908666380610404, "Regression_sin_phi": 0.04756067963306791, "Regression_cos_phi": 0.04790033384485476, "Regression_energy": 0.41244882345199585, "Total": 13.508974326063436, "Total_unweighted": 4.427989109169294}}, "valid": {"Classification_binary": 3.1929747263590493, "Classification": 0.014891543736060461, "Regression_pt": 0.29772748549779254, "Regression_eta": 0.08769792318344116, "Regression_sin_phi": 0.047768567999204, "Regression_cos_phi": 0.04763929049173991, "Regression_energy": 0.3435819943745931, "Total": 12.675401051839193, "group/physical_pu/Classification": 0.015492431819438934, "group/physical_pu/Classification_binary": 2.9142675399780273, "group/physical_pu/Regression_cos_phi": 0.04733716448148092, "group/physical_pu/Regression_energy": 0.2743512988090515, "group/physical_pu/Regression_eta": 0.10464646418889363, "group/physical_pu/Regression_pt": 0.22777088483174643, "group/physical_pu/Regression_sin_phi": 0.04727142552534739, "group/physical_pu/Total": 11.770997365315756, "group/physical_pu/Total_unweighted": 3.631137209633986, "group/physical_nopu/Classification": 0.014290654410918554, "group/physical_nopu/Classification_binary": 3.471681594848633, "group/physical_nopu/Regression_cos_phi": 0.0479414165019989, "group/physical_nopu/Regression_energy": 0.4128127892812093, "group/physical_nopu/Regression_eta": 0.0707493672768275, "group/physical_nopu/Regression_pt": 0.3676840464274089, "group/physical_nopu/Regression_sin_phi": 0.04826569060484568, "group/physical_nopu/Total": 13.579803466796875, "group/physical_nopu/Total_unweighted": 4.433425559351842, "Total_unweighted": 4.03228153164188, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.014716163277626038, "diagnostic/loss/cms_pf/Classification_binary": 3.190532684326172, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.04733486846089363, "diagnostic/loss/cms_pf/Regression_energy": 0.3306228518486023, "diagnostic/loss/cms_pf/Regression_eta": 0.0877683088183403, "diagnostic/loss/cms_pf/Regression_pt": 0.2847622036933899, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.046049632132053375, "diagnostic/regression/cms_pf/energy_pred_mean": 0.19783693552017212, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.3514072299003601, "diagnostic/regression/cms_pf/energy_residual_mean": 0.08707071095705032, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5820175409317017, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.11154396086931229, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.32467928528785706, "diagnostic/regression/cms_pf/pt_residual_mean": 0.1223161444067955, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5424451231956482, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8988738656044006, "metrics/particle/class_1/pid_accuracy": 0.978965699672699, "metrics/particle/class_2/efficiency": 0.25708216428756714, "metrics/particle/class_2/pid_accuracy": 0.2058691680431366, "metrics/particle/class_3/efficiency": 0.3977532684803009, "metrics/particle/class_3/pid_accuracy": 0.4786675274372101, "metrics/particle/class_4/efficiency": 0.4411793649196625, "metrics/particle/class_4/pid_accuracy": 0.6635446548461914, "metrics/particle/class_5/efficiency": 0.37574082612991333, "metrics/particle/class_5/pid_accuracy": 0.6335993409156799, "metrics/particle/class_6/efficiency": 0.8675074577331543, "metrics/particle/class_6/pid_accuracy": 0.061907872557640076, "metrics/particle/class_7/efficiency": 0.965448796749115, "metrics/particle/class_7/pid_accuracy": 0.24105305969715118, "metrics/particle/count/bias_mean": 69.93033599853516, "metrics/particle/count/mae": 92.013671875, "metrics/particle/count/prediction_mean": 1598.5338134765625, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.08554051071405411, "metrics/particle/event/energy_response_mean": 0.9307606816291809, "metrics/particle/event/met_abs_error": 19.520713806152344, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04844934865832329, "metrics/particle/event/scalar_pt_response_mean": 1.0055500268936157, "metrics/particle/event/vector_pt_closure": 0.02764890342950821, "metrics/particle/matched/delta_eta_abs_mean": 0.018641270697116852, "metrics/particle/matched/delta_phi_abs_mean": 0.015976114198565483, "metrics/particle/matched/delta_r_mean": 0.02753622829914093, "metrics/particle/matched/energy_relative_abs_mean": 0.11324173212051392, "metrics/particle/matched/pid_accuracy": 0.8477852940559387, "metrics/particle/matched/pt_relative_abs_mean": 0.1045932024717331, "metrics/particle/matching/duplicate_fraction": 0.10905073583126068, "metrics/particle/matching/efficiency": 0.6420096755027771, "metrics/particle/matching/f1": 0.6276528239250183, "metrics/particle/matching/purity": 0.613923966884613, "metrics/particle/matching/target_coverage_dr0p05": 0.5174904465675354, "metrics/particle/matching/target_coverage_dr0p10": 0.7042428851127625, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5610873103141785, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7597152590751648, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01707063429057598, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06628254055976868, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.058030299842357635, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0036322614178061485, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.11028893291950226, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0018213619478046894, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.2171742171049118, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.010600184090435505, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.263246774673462, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 69179.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4595644474029541, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.090996503829956, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.36873823404312134, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.24717779457569122, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13116998970508575, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.8246073722839355, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03974122181534767, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8171029090881348, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.05855155363678932, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.376636505126953, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87642.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7892551422119141, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 37.536903381347656, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 43.48017120361328, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 16.562891006469727, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.5474112033843994, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.2309203147888184, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 38.396446228027344, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.7735953330993652, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 60.2879524230957, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04035589098930359, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16022835671901703, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15764069557189941, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.014165043830871582, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2115200012922287, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17235049605369568, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.457042217254639, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.010499759577214718, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.56930160522461, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 122042.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5106223821640015, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9485151767730713, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9336230754852295, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7178519368171692, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3150734007358551, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3.724581003189087, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.3740091323852539, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.60158920288086, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.10055840015411377, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.00053787231445, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228588.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5276100039482117, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -29.43581771850586, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 375.1967468261719, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 19.38467788696289, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.2923891544342041, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.43166062235832214, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 373.23175048828125, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -66.1428451538086, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 560.3308715820312, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013056265190243721, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.07133091241121292, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.049692537635564804, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01203700341284275, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10778787732124329, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007850604597479105, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.07958892732858658, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0005862669786438346, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8112684488296509, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59868.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3333820104598999, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.403666615486145, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.26600196957588196, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.17335641384124756, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10341722518205643, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 3.910067081451416, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.023160140961408615, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0863229036331177, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0464383102953434, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.755603790283203, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72193.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8292196393013, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.6055564880371094, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 20.66438865661621, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 15.57989501953125, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 3.2408080101013184, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.8767945170402527, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 11.265263557434082, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.29734575748443604, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 22.659013748168945, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.027431290596723557, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11314868927001953, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10445800423622131, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.003014160320162773, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1669362336397171, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08446009457111359, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.7312111854553223, "metrics/particle/momentum/all/accepted/pz_bias": 0.007884150370955467, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.102707862854004, "metrics/particle/momentum/all/accepted/samples": 251089.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0546625852584839, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.065259337425232, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6820798516273499, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5104503035545349, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.23423947393894196, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.5785176753997803, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.23337700963020325, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.793485641479492, "metrics/particle/momentum/all/assigned/pz_bias": -0.08102130889892578, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.119070053100586, "metrics/particle/momentum/all/assigned/samples": 388423.0, "metrics/particle/momentum/all/efficiency": 0.6419087052345276, "metrics/particle/momentum/all/event/energy_bias": 5.495532989501953, "metrics/particle/momentum/all/event/p3_error_mean": 377.5415344238281, "metrics/particle/momentum/all/event/pt_vector_error_mean": 26.838218688964844, "metrics/particle/momentum/all/event/px_bias": 4.495830059051514, "metrics/particle/momentum/all/event/py_bias": 0.7857863306999207, "metrics/particle/momentum/all/event/pz_abs_mean": 373.5999755859375, "metrics/particle/momentum/all/event/pz_bias": -63.66659164428711, "metrics/particle/momentum/all/event/pz_residual_rms": 559.9734497070312, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.013663598336279392, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06144581362605095, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05108002573251724, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009798988699913025, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0960112065076828, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001170946634374559, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1990383416414261, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0027228554245084524, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.145691990852356, "metrics/particle/momentum/class_1/accepted/samples": 173364.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.2136267125606537, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.8866825103759766, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.30361369252204895, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24905143678188324, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06967238336801529, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.4640698432922363, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0222887322306633, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1058416366577148, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.006262360140681267, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.454414367675781, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8992468118667603, "metrics/particle/momentum/class_1/event/energy_bias": 51.70713806152344, "metrics/particle/momentum/class_1/event/p3_error_mean": 54.483726501464844, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 16.01925277709961, "metrics/particle/momentum/class_1/event/px_bias": -0.04566901922225952, "metrics/particle/momentum/class_1/event/py_bias": -0.5855883359909058, "metrics/particle/momentum/class_1/event/pz_abs_mean": 49.18894577026367, "metrics/particle/momentum/class_1/event/pz_bias": 1.9281415939331055, "metrics/particle/momentum/class_1/event/pz_residual_rms": 82.40233612060547, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.058158110827207565, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2402212917804718, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23226776719093323, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07138034701347351, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27266770601272583, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.11130719631910324, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.230152606964111, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.07555661350488663, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 10.27176284790039, "metrics/particle/momentum/class_2/accepted/samples": 7933.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.2062368392944336, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.5036667585372925, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.11846125125885, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6352466344833374, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.47108662128448486, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.161643028259277, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.35437002778053284, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.654067993164062, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.1036173552274704, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 34.520931243896484, "metrics/particle/momentum/class_2/assigned/samples": 31041.0, "metrics/particle/momentum/class_2/efficiency": 0.25535956025123596, "metrics/particle/momentum/class_2/event/energy_bias": -265.47320556640625, "metrics/particle/momentum/class_2/event/p3_error_mean": 165.199951171875, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 28.759904861450195, "metrics/particle/momentum/class_2/event/px_bias": 2.458202362060547, "metrics/particle/momentum/class_2/event/py_bias": -2.4269046783447266, "metrics/particle/momentum/class_2/event/pz_abs_mean": 159.26193237304688, "metrics/particle/momentum/class_2/event/pz_bias": -16.91330337524414, "metrics/particle/momentum/class_2/event/pz_residual_rms": 221.56741333007812, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.06052066758275032, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.22614535689353943, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.2226983606815338, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.007444617338478565, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.26237568259239197, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.30921798944473267, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.273768424987793, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.010927380062639713, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 13.919891357421875, "metrics/particle/momentum/class_3/accepted/samples": 21176.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.018108606338501, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.9318920373916626, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.0623893737792969, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.8811978697776794, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.41616326570510864, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 2.589770555496216, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5294890999794006, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.10710334777832, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.07411141693592072, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.215110778808594, "metrics/particle/momentum/class_3/assigned/samples": 52477.0, "metrics/particle/momentum/class_3/efficiency": 0.3972908556461334, "metrics/particle/momentum/class_3/event/energy_bias": 443.11737060546875, "metrics/particle/momentum/class_3/event/p3_error_mean": 285.0924072265625, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.550427436828613, "metrics/particle/momentum/class_3/event/px_bias": 0.046487435698509216, "metrics/particle/momentum/class_3/event/py_bias": 0.016938775777816772, "metrics/particle/momentum/class_3/event/pz_abs_mean": 284.6483154296875, "metrics/particle/momentum/class_3/event/pz_bias": -35.51879119873047, "metrics/particle/momentum/class_3/event/pz_residual_rms": 452.20880126953125, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.0596945695579052, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.23122462630271912, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22786803543567657, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.057524457573890686, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.26696136593818665, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.3589895963668823, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 10.972748756408691, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.037352994084358215, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.20247459411621, "metrics/particle/momentum/class_4/accepted/samples": 37015.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.8501909971237183, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0242365598678589, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.053138017654419, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.7763838768005371, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3639342188835144, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.2161757946014404, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5244948863983154, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.199695587158203, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.28288301825523376, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 39.71893310546875, "metrics/particle/momentum/class_4/assigned/samples": 82405.0, "metrics/particle/momentum/class_4/efficiency": 0.4391699433326721, "metrics/particle/momentum/class_4/event/energy_bias": -179.510986328125, "metrics/particle/momentum/class_4/event/p3_error_mean": 321.59033203125, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 10.97377872467041, "metrics/particle/momentum/class_4/event/px_bias": -0.44575199484825134, "metrics/particle/momentum/class_4/event/py_bias": 0.4145626723766327, "metrics/particle/momentum/class_4/event/pz_abs_mean": 321.0476989746094, "metrics/particle/momentum/class_4/event/pz_bias": -17.555545806884766, "metrics/particle/momentum/class_4/event/pz_residual_rms": 472.2655029296875, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.05093790590763092, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22485075891017914, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21342749893665314, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.037518054246902466, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2540317177772522, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.025657473132014275, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.351034164428711, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.011832204647362232, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.726724624633789, "metrics/particle/momentum/class_5/accepted/samples": 10913.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4308266639709473, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.149508237838745, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9941776394844055, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6918767690658569, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.38126340508461, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 5.605262756347656, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.14907734096050262, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.48992395401001, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.07115793973207474, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 18.04140281677246, "metrics/particle/momentum/class_5/assigned/samples": 28938.0, "metrics/particle/momentum/class_5/efficiency": 0.3769993484020233, "metrics/particle/momentum/class_5/event/energy_bias": -34.82942581176758, "metrics/particle/momentum/class_5/event/p3_error_mean": 112.75118255615234, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 25.511856079101562, "metrics/particle/momentum/class_5/event/px_bias": 3.2126832008361816, "metrics/particle/momentum/class_5/event/py_bias": 3.744586229324341, "metrics/particle/momentum/class_5/event/pz_abs_mean": 104.48731231689453, "metrics/particle/momentum/class_5/event/pz_bias": 5.807872772216797, "metrics/particle/momentum/class_5/event/pz_residual_rms": 163.2338409423828, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.016686489805579185, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.08564198762178421, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.08629591017961502, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.030869558453559875, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.1277158260345459, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.0060728746466338634, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.6382712721824646, "metrics/particle/momentum/class_6/accepted/pz_bias": -0.13531093299388885, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 3.046640634536743, "metrics/particle/momentum/class_6/accepted/samples": 494.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.3541872799396515, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 1.0940216779708862, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.6375502347946167, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.501329779624939, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.10782608389854431, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 4.06323766708374, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.05043478310108185, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 2.4383111000061035, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.28002405166625977, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 10.323867797851562, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8591304421424866, "metrics/particle/momentum/class_6/event/energy_bias": -8.24467658996582, "metrics/particle/momentum/class_6/event/p3_error_mean": 6.849165439605713, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 2.576033115386963, "metrics/particle/momentum/class_6/event/px_bias": -0.5596170425415039, "metrics/particle/momentum/class_6/event/py_bias": -0.3506358861923218, "metrics/particle/momentum/class_6/event/pz_abs_mean": 5.972554683685303, "metrics/particle/momentum/class_6/event/pz_bias": -1.28077232837677, "metrics/particle/momentum/class_6/event/pz_residual_rms": 17.507448196411133, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.011557129211723804, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.043734289705753326, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.04163268581032753, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.017402783036231995, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.08205406367778778, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.010309278033673763, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.6000403761863708, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.2426125407218933, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 2.4023642539978027, "metrics/particle/momentum/class_7/accepted/samples": 194.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.04277859628200531, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.08795560896396637, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.06689193844795227, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.0375567302107811, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.02512562833726406, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.20171435177326202, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.01005025114864111, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.6506872177124023, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.20681409537792206, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 2.425363540649414, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.9748743772506714, "metrics/particle/momentum/class_7/event/energy_bias": -1.270709753036499, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.118936777114868, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 0.9077147245407104, "metrics/particle/momentum/class_7/event/px_bias": -0.17050494253635406, "metrics/particle/momentum/class_7/event/py_bias": -0.02717190608382225, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.7535051107406616, "metrics/particle/momentum/class_7/event/pz_bias": -0.1341954469680786, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.763657093048096, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 3.5646414756774902, "step_valid_time": 17225.78834080696, "step_total_time": 17229.352982759476, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.031647762625813, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.34586762817740346, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81545, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15925, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13106, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19545872967167843, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19529094365074498, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19537480063795853, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.804709056349255, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1608591592513041, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.16072107425347967, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.16079008710587658, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19545872967167843, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0234610408155635, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3564176747492832, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81454, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15476, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12570, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19049963687391525, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18999680801433938, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19024789019810318, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8100031919856606, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15472864018513274, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15432022982296756, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15452416514539655, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19049963687391525, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0210367062627443, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3837698859943345, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 80857, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 14594, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11797, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18064340442386959, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18049148496728792, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18056741274142263, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.819508515032712, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14602235452846304, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1458995510592775, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.1459609269638593, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18064340442386959} \ No newline at end of file diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_80000.json b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_80000.json new file mode 100644 index 0000000000000000000000000000000000000000..b3b43acbc832fe3c03f0375f870f3489a7a9fc15 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_80000.json @@ -0,0 +1 @@ +{"epoch": 0.34474140085668237, "train": {"Classification_binary": 3.4149365425109863, "Classification": 0.01547944825142622, "Regression_pt": 0.583466649055481, "Regression_eta": 0.08007072657346725, "Regression_sin_phi": 0.04651808738708496, "Regression_cos_phi": 0.04528043419122696, "Regression_energy": 0.6188869476318359, "Total": 16.1833438873291, "Total_unweighted": 4.804638835601509}, "train_group": {"physical_pu": {"Classification_binary": 2.9065479708241893, "Classification": 0.01553585542017942, "Regression_pt": 0.22749776401362576, "Regression_eta": 0.1053000569834814, "Regression_sin_phi": 0.047094785332024755, "Regression_cos_phi": 0.04712688746851879, "Regression_energy": 0.2741471005010081, "Total": 11.756600736261724, "Total_unweighted": 3.6232504205430276}, "physical_nopu": {"Classification_binary": 3.464570965310528, "Classification": 0.014205006024993883, "Regression_pt": 0.36197215043278813, "Regression_eta": 0.06904683687505943, "Regression_sin_phi": 0.04771541858171866, "Regression_cos_phi": 0.04786722316141307, "Regression_energy": 0.40880449888336384, "Total": 13.456669942774, "Total_unweighted": 4.414182099269865}}, "valid": {"Classification_binary": 3.1854785283406577, "Classification": 0.01486896350979805, "Regression_pt": 0.2960307002067566, "Regression_eta": 0.08764699101448059, "Regression_sin_phi": 0.04772174855073293, "Regression_cos_phi": 0.047594815492630005, "Regression_energy": 0.34196341037750244, "Total": 12.642045338948568, "group/physical_pu/Classification": 0.015477704505125681, "group/physical_pu/Classification_binary": 2.9076321919759116, "group/physical_pu/Regression_cos_phi": 0.04731094837188721, "group/physical_pu/Regression_energy": 0.27380166451136273, "group/physical_pu/Regression_eta": 0.1045525074005127, "group/physical_pu/Regression_pt": 0.22719168663024902, "group/physical_pu/Regression_sin_phi": 0.047247931361198425, "group/physical_pu/Total": 11.753445943196615, "group/physical_pu/Total_unweighted": 3.6232146347562475, "group/physical_nopu/Classification": 0.01426022375623385, "group/physical_nopu/Classification_binary": 3.463324864705404, "group/physical_nopu/Regression_cos_phi": 0.047878687580426536, "group/physical_nopu/Regression_energy": 0.41012513637542725, "group/physical_nopu/Regression_eta": 0.07074146966139476, "group/physical_nopu/Regression_pt": 0.36486971378326416, "group/physical_nopu/Regression_sin_phi": 0.04819556574026743, "group/physical_nopu/Total": 13.530643463134766, "group/physical_nopu/Total_unweighted": 4.419395661602418, "Total_unweighted": 4.0213051574925585, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.014689825475215912, "diagnostic/loss/cms_pf/Classification_binary": 3.1837968826293945, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.04725345969200134, "diagnostic/loss/cms_pf/Regression_energy": 0.3279876410961151, "diagnostic/loss/cms_pf/Regression_eta": 0.08774149417877197, "diagnostic/loss/cms_pf/Regression_pt": 0.2818683981895447, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.046052780002355576, "diagnostic/regression/cms_pf/energy_pred_mean": 0.20187626779079437, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.3520151674747467, "diagnostic/regression/cms_pf/energy_residual_mean": 0.09111005812883377, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5832304954528809, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.11321085691452026, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.324436217546463, "diagnostic/regression/cms_pf/pt_residual_mean": 0.12398304045200348, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5430651307106018, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8975579738616943, "metrics/particle/class_1/pid_accuracy": 0.978687047958374, "metrics/particle/class_2/efficiency": 0.2563909590244293, "metrics/particle/class_2/pid_accuracy": 0.20834682881832123, "metrics/particle/class_3/efficiency": 0.3924287259578705, "metrics/particle/class_3/pid_accuracy": 0.47749269008636475, "metrics/particle/class_4/efficiency": 0.43883371353149414, "metrics/particle/class_4/pid_accuracy": 0.6633965969085693, "metrics/particle/class_5/efficiency": 0.3729006350040436, "metrics/particle/class_5/pid_accuracy": 0.6353753805160522, "metrics/particle/class_6/efficiency": 0.8604222536087036, "metrics/particle/class_6/pid_accuracy": 0.06044137105345726, "metrics/particle/class_7/efficiency": 0.9670373201370239, "metrics/particle/class_7/pid_accuracy": 0.24229979515075684, "metrics/particle/count/bias_mean": 61.84309768676758, "metrics/particle/count/mae": 84.48697662353516, "metrics/particle/count/prediction_mean": 1590.4466552734375, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.08769464492797852, "metrics/particle/event/energy_response_mean": 0.9406366944313049, "metrics/particle/event/met_abs_error": 18.93034553527832, "metrics/particle/event/scalar_pt_relative_abs_error": 0.048748716711997986, "metrics/particle/event/scalar_pt_response_mean": 1.0044407844543457, "metrics/particle/event/vector_pt_closure": 0.027189305052161217, "metrics/particle/matched/delta_eta_abs_mean": 0.018973957747220993, "metrics/particle/matched/delta_phi_abs_mean": 0.01579345017671585, "metrics/particle/matched/delta_r_mean": 0.02772163785994053, "metrics/particle/matched/energy_relative_abs_mean": 0.113133005797863, "metrics/particle/matched/pid_accuracy": 0.8481951951980591, "metrics/particle/matched/pt_relative_abs_mean": 0.10387390106916428, "metrics/particle/matching/duplicate_fraction": 0.10803540796041489, "metrics/particle/matching/efficiency": 0.639853298664093, "metrics/particle/matching/f1": 0.6271666288375854, "metrics/particle/matching/purity": 0.6149731874465942, "metrics/particle/matching/target_coverage_dr0p05": 0.5145063400268555, "metrics/particle/matching/target_coverage_dr0p10": 0.7015283703804016, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5610954165458679, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7580731511116028, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017353828996419907, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06667222827672958, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.05783594027161598, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0007365340716205537, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.11044137924909592, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.001841487130150199, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.21596623957157135, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.00879648607224226, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2997139692306519, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 68966.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4681371748447418, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1327450275421143, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.37782183289527893, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.2522335648536682, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1319884955883026, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.3396522998809814, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03974031284451485, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8196403980255127, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.07475126534700394, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.280438423156738, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87644.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7868250012397766, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 30.411426544189453, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 41.02257537841797, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 15.510160446166992, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.5851926803588867, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.1486092060804367, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 36.16386032104492, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.3333890438079834, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 54.23004150390625, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.040689677000045776, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1595868021249771, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1569078415632248, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.014994094148278236, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21061065793037415, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17350643873214722, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.467339992523193, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.016441931948065758, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.531633377075195, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 121471.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5162889957427979, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9722626805305481, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9445203542709351, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7326527237892151, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3182707130908966, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3.6980273723602295, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.3780514895915985, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.697338104248047, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.04803696274757385, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.04425048828125, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228535.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5251414775848389, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 314.3128662109375, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 368.84429931640625, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 19.110498428344727, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -1.3120442628860474, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -1.8195828199386597, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 367.0798645019531, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -33.297237396240234, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 557.6192016601562, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012878633104264736, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.07213535904884338, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.04892410710453987, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008899860084056854, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10659091174602509, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000852700206451118, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.07882891595363617, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0011827998096123338, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8091097474098206, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 59810.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.33579733967781067, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.7349492311477661, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.25475814938545227, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.15989279747009277, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10226894915103912, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 3.054952621459961, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.022731050848960876, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.067639946937561, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.049957625567913055, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.384222030639648, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72192.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8284162878990173, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -8.30819034576416, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 19.006664276123047, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 14.171662330627441, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.477579116821289, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.8762747049331665, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 10.79775333404541, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.8409441113471985, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 19.447481155395508, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.027611564844846725, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11307907104492188, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10379591584205627, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.005354081280529499, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16615641117095947, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08493208885192871, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.732229471206665, "metrics/particle/momentum/all/accepted/pz_bias": 0.010122532024979591, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.072818756103516, "metrics/particle/momentum/all/accepted/samples": 250247.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.060317039489746, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1502501964569092, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6884171366691589, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5177690982818604, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.23608095943927765, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.3192391395568848, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.23565611243247986, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.846117973327637, "metrics/particle/momentum/all/assigned/pz_bias": -0.05442262068390846, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.124649047851562, "metrics/particle/momentum/all/assigned/samples": 388371.0, "metrics/particle/momentum/all/efficiency": 0.6397560834884644, "metrics/particle/momentum/all/event/energy_bias": 336.4161071777344, "metrics/particle/momentum/all/event/p3_error_mean": 371.7464294433594, "metrics/particle/momentum/all/event/pt_vector_error_mean": 25.570159912109375, "metrics/particle/momentum/all/event/px_bias": 0.750727653503418, "metrics/particle/momentum/all/event/py_bias": -3.8444666862487793, "metrics/particle/momentum/all/event/pz_abs_mean": 368.37872314453125, "metrics/particle/momentum/all/event/pz_bias": -32.78968048095703, "metrics/particle/momentum/all/event/pz_residual_rms": 557.0092163085938, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01396325696259737, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06185800954699516, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05066287890076637, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005160349886864424, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09531447291374207, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012188077671453357, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.19848230481147766, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.000678260694257915, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.0399309396743774, "metrics/particle/momentum/class_1/accepted/samples": 173120.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.221574068069458, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.0294570922851562, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3045065104961395, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24578078091144562, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06993692368268967, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.0122270584106445, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.022890428081154823, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1263879537582397, "metrics/particle/momentum/class_1/assigned/pz_bias": 3.625612635005382e-06, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.463394641876221, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8979812264442444, "metrics/particle/momentum/class_1/event/energy_bias": 22.466068267822266, "metrics/particle/momentum/class_1/event/p3_error_mean": 52.51650619506836, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.623668670654297, "metrics/particle/momentum/class_1/event/px_bias": 0.18297487497329712, "metrics/particle/momentum/class_1/event/py_bias": -1.0530204772949219, "metrics/particle/momentum/class_1/event/pz_abs_mean": 47.20528030395508, "metrics/particle/momentum/class_1/event/pz_bias": -2.990377426147461, "metrics/particle/momentum/class_1/event/pz_residual_rms": 78.25059509277344, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.05802108347415924, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.24180462956428528, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2330954670906067, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07320217043161392, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2734376788139343, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.11244019120931625, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.214722156524658, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.10292473435401917, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 9.72955322265625, "metrics/particle/momentum/class_2/accepted/samples": 7942.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.1920530796051025, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.5186594724655151, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1403751373291016, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6552145481109619, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.47026801109313965, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.21632194519043, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.35298284888267517, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.550840377807617, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.053992338478565216, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 34.3475227355957, "metrics/particle/momentum/class_2/assigned/samples": 31044.0, "metrics/particle/momentum/class_2/efficiency": 0.25564926862716675, "metrics/particle/momentum/class_2/event/energy_bias": -289.0379943847656, "metrics/particle/momentum/class_2/event/p3_error_mean": 165.53456115722656, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 28.589906692504883, "metrics/particle/momentum/class_2/event/px_bias": 2.3298499584198, "metrics/particle/momentum/class_2/event/py_bias": -3.987614154815674, "metrics/particle/momentum/class_2/event/pz_abs_mean": 159.66250610351562, "metrics/particle/momentum/class_2/event/pz_bias": -9.727998733520508, "metrics/particle/momentum/class_2/event/pz_residual_rms": 222.8994598388672, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.060707107186317444, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.22599931061267853, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.22264713048934937, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.0049788206815719604, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.26221051812171936, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.31366637349128723, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.385520935058594, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.05116214603185654, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 14.017669677734375, "metrics/particle/momentum/class_3/accepted/samples": 20898.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.0295250415802, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.9842368364334106, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.0979156494140625, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.9230430722236633, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4230307340621948, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 2.7600271701812744, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5363547801971436, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.473608016967773, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.1061970666050911, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.704891204833984, "metrics/particle/momentum/class_3/assigned/samples": 52469.0, "metrics/particle/momentum/class_3/efficiency": 0.39207518100738525, "metrics/particle/momentum/class_3/event/energy_bias": 470.1885681152344, "metrics/particle/momentum/class_3/event/p3_error_mean": 286.0863037109375, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.583023071289062, "metrics/particle/momentum/class_3/event/px_bias": -0.22822128236293793, "metrics/particle/momentum/class_3/event/py_bias": -0.5294697284698486, "metrics/particle/momentum/class_3/event/pz_abs_mean": 285.6575012207031, "metrics/particle/momentum/class_3/event/pz_bias": -34.580806732177734, "metrics/particle/momentum/class_3/event/pz_residual_rms": 457.1041564941406, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.059828221797943115, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.2302750200033188, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.2270609587430954, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.05240499600768089, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.26605165004730225, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.3623582422733307, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 11.017578125, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.07157986611127853, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.259164810180664, "metrics/particle/momentum/class_4/accepted/samples": 36773.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.8537708520889282, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0787296295166016, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.066217303276062, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8003158569335938, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3679213225841522, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.1856913566589355, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5304880738258362, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.23529815673828, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.12515969574451447, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 39.57423782348633, "metrics/particle/momentum/class_4/assigned/samples": 82360.0, "metrics/particle/momentum/class_4/efficiency": 0.4362986981868744, "metrics/particle/momentum/class_4/event/energy_bias": 159.50645446777344, "metrics/particle/momentum/class_4/event/p3_error_mean": 324.16461181640625, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 10.785661697387695, "metrics/particle/momentum/class_4/event/px_bias": -1.1255474090576172, "metrics/particle/momentum/class_4/event/py_bias": -0.39707228541374207, "metrics/particle/momentum/class_4/event/pz_abs_mean": 323.74627685546875, "metrics/particle/momentum/class_4/event/pz_bias": 24.232769012451172, "metrics/particle/momentum/class_4/event/pz_residual_rms": 473.08807373046875, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.05097578465938568, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2241869419813156, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2123945951461792, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03645043820142746, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.25297269225120544, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.024194292724132538, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.3139837980270386, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.005798094440251589, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.312875747680664, "metrics/particle/momentum/class_5/accepted/samples": 10829.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4392729997634888, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.0712459087371826, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9472011923789978, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6458574533462524, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.38142797350883484, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.884235382080078, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.14829277992248535, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.430469989776611, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.23400971293449402, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 17.783964157104492, "metrics/particle/momentum/class_5/assigned/samples": 28936.0, "metrics/particle/momentum/class_5/efficiency": 0.3740974962711334, "metrics/particle/momentum/class_5/event/energy_bias": -16.581058502197266, "metrics/particle/momentum/class_5/event/p3_error_mean": 112.99061584472656, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 25.186073303222656, "metrics/particle/momentum/class_5/event/px_bias": 0.36943748593330383, "metrics/particle/momentum/class_5/event/py_bias": 2.769354820251465, "metrics/particle/momentum/class_5/event/pz_abs_mean": 105.00947570800781, "metrics/particle/momentum/class_5/event/pz_bias": -8.616448402404785, "metrics/particle/momentum/class_5/event/pz_residual_rms": 162.51800537109375, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.017230544239282608, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.08539904654026031, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.08671234548091888, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.03683945909142494, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.1300102323293686, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.01224489789456129, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.5436803698539734, "metrics/particle/momentum/class_6/accepted/pz_bias": 0.06446979194879532, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 2.1021928787231445, "metrics/particle/momentum/class_6/accepted/samples": 490.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.36105915904045105, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 1.1490360498428345, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.7160969376564026, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.5592424273490906, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.11652173846960068, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 4.703455448150635, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.04173913225531578, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 2.1825640201568604, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.2084062695503235, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 9.388299942016602, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.852173924446106, "metrics/particle/momentum/class_6/event/energy_bias": -8.71391773223877, "metrics/particle/momentum/class_6/event/p3_error_mean": 7.119056701660156, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 2.7835681438446045, "metrics/particle/momentum/class_6/event/px_bias": -0.6552503108978271, "metrics/particle/momentum/class_6/event/py_bias": -0.5415544509887695, "metrics/particle/momentum/class_6/event/pz_abs_mean": 6.111410617828369, "metrics/particle/momentum/class_6/event/pz_bias": -0.8154345750808716, "metrics/particle/momentum/class_6/event/pz_residual_rms": 17.617799758911133, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.012350879609584808, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.04115542396903038, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.03852040693163872, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.013511733151972294, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.07828022539615631, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.010256410576403141, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.5853619575500488, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.3031442165374756, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 2.838510513305664, "metrics/particle/momentum/class_7/accepted/samples": 195.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.05720892921090126, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.15667036175727844, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.0707915648818016, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.03617437556385994, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.015075377188622952, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.3467261791229248, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.015075377188622952, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 0.6658896207809448, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.20934708416461945, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 2.931413173675537, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.979899525642395, "metrics/particle/momentum/class_7/event/energy_bias": -1.4119664430618286, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.150365114212036, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 0.9184801578521729, "metrics/particle/momentum/class_7/event/px_bias": -0.12251587212085724, "metrics/particle/momentum/class_7/event/py_bias": -0.10508961975574493, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.783874750137329, "metrics/particle/momentum/class_7/event/pz_bias": -0.2913879156112671, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.910571575164795, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 4.58496618270874, "step_valid_time": 19758.324454545975, "step_total_time": 19762.90942120552, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0272258580637836, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.34276301689871147, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81584, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15876, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13098, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19485731819576557, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19459697979996077, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.19472706198369916, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8054030202000392, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.16076096962258363, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.16054618552657385, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.16065350578624915, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19485731819576557, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.02281342955671, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3586737651567595, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81286, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15468, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12576, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.19040116200347124, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.19029107103314225, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.1903461005999077, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8097089289668578, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.15480249633796575, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.15471298870654232, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15475772958006462, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.19040116200347124, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0204450555993785, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3842559148934941, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 80993, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 14572, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11758, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.18037109012365546, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.1799167829318583, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18014365009704417, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8200832170681417, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1455396155417198, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14517303964540146, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14535609647550407, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.18037109012365546} \ No newline at end of file diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_90000.json b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_90000.json new file mode 100644 index 0000000000000000000000000000000000000000..15de2f33ba87f25a6ffa755fa7bf7926999a9e28 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/history/step_90000.json @@ -0,0 +1 @@ +{"epoch": 0.3878340759637677, "train": {"Classification_binary": 2.9177095890045166, "Classification": 0.014874441549181938, "Regression_pt": 0.21554072201251984, "Regression_eta": 0.10398383438587189, "Regression_sin_phi": 0.04701986536383629, "Regression_cos_phi": 0.04752826690673828, "Regression_energy": 0.2660740911960602, "Total": 11.610236167907715, "Total_unweighted": 3.612730810418725}, "train_group": {"physical_pu": {"Classification_binary": 2.9067262293218254, "Classification": 0.015493557972649296, "Regression_pt": 0.22679426529250302, "Regression_eta": 0.1045960281248931, "Regression_sin_phi": 0.047077174411042706, "Regression_cos_phi": 0.0471664778478853, "Regression_energy": 0.2731043503179655, "Total": 11.734834775819884, "Total_unweighted": 3.6209580832887647}, "physical_nopu": {"Classification_binary": 3.460074945907257, "Classification": 0.014211808423520905, "Regression_pt": 0.36291235680430634, "Regression_eta": 0.06878145284975322, "Regression_sin_phi": 0.04767268649799333, "Regression_cos_phi": 0.047938915848633634, "Regression_energy": 0.40927771261685225, "Total": 13.458554430501033, "Total_unweighted": 4.410869878948317}}, "valid": {"Classification_binary": 3.1811917622884116, "Classification": 0.014844149351119995, "Regression_pt": 0.29521199067433673, "Regression_eta": 0.08760765194892883, "Regression_sin_phi": 0.04769539336363474, "Regression_cos_phi": 0.047552913427352905, "Regression_energy": 0.3413265546162923, "Total": 12.624483744303385, "group/physical_pu/Classification": 0.015466084082921347, "group/physical_pu/Classification_binary": 2.904437700907389, "group/physical_pu/Regression_cos_phi": 0.047290479143460594, "group/physical_pu/Regression_energy": 0.2737988630930583, "group/physical_pu/Regression_eta": 0.10447264711062114, "group/physical_pu/Regression_pt": 0.22687027851740518, "group/physical_pu/Regression_sin_phi": 0.047231401006380715, "group/physical_pu/Total": 11.744926452636719, "group/physical_pu/Total_unweighted": 3.6195674538612366, "group/physical_nopu/Classification": 0.014222212135791779, "group/physical_nopu/Classification_binary": 3.4579458236694336, "group/physical_nopu/Regression_cos_phi": 0.047815337777137756, "group/physical_nopu/Regression_energy": 0.4088541666666667, "group/physical_nopu/Regression_eta": 0.070742666721344, "group/physical_nopu/Regression_pt": 0.3635537624359131, "group/physical_nopu/Regression_sin_phi": 0.04815937578678131, "group/physical_nopu/Total": 13.504038492838541, "group/physical_nopu/Total_unweighted": 4.4112933451930685, "Total_unweighted": 4.015430415670077, "diagnostic/composition/cms_pf/events": 8.0, "diagnostic/composition/cms_pf/target_particles": 12223.75, "diagnostic/composition/cms_pf/valid_elements": 16529.78125, "diagnostic/loss/cms_pf/Classification": 0.014656363055109978, "diagnostic/loss/cms_pf/Classification_binary": 3.1785061359405518, "diagnostic/loss/cms_pf/Regression_cos_phi": 0.04718054458498955, "diagnostic/loss/cms_pf/Regression_energy": 0.32964056730270386, "diagnostic/loss/cms_pf/Regression_eta": 0.0877435952425003, "diagnostic/loss/cms_pf/Regression_pt": 0.28366905450820923, "diagnostic/loss/cms_pf/Regression_sin_phi": 0.045971449464559555, "diagnostic/regression/cms_pf/energy_pred_mean": 0.19947479665279388, "diagnostic/regression/cms_pf/energy_residual_abs_mean": 0.3510538637638092, "diagnostic/regression/cms_pf/energy_residual_mean": 0.08870858699083328, "diagnostic/regression/cms_pf/energy_residual_rms": 0.5820817351341248, "diagnostic/regression/cms_pf/energy_target_mean": 0.1107662171125412, "diagnostic/regression/cms_pf/pt_pred_mean": 0.11050863564014435, "diagnostic/regression/cms_pf/pt_residual_abs_mean": 0.32333794236183167, "diagnostic/regression/cms_pf/pt_residual_mean": 0.12128081172704697, "diagnostic/regression/cms_pf/pt_residual_rms": 0.5411434769630432, "diagnostic/regression/cms_pf/pt_target_mean": -0.010772179812192917, "metrics/particle/class_1/efficiency": 0.8993476629257202, "metrics/particle/class_1/pid_accuracy": 0.9788728952407837, "metrics/particle/class_2/efficiency": 0.25563812255859375, "metrics/particle/class_2/pid_accuracy": 0.20951582491397858, "metrics/particle/class_3/efficiency": 0.39397868514060974, "metrics/particle/class_3/pid_accuracy": 0.4754721224308014, "metrics/particle/class_4/efficiency": 0.43886420130729675, "metrics/particle/class_4/pid_accuracy": 0.6618189811706543, "metrics/particle/class_5/efficiency": 0.3775111734867096, "metrics/particle/class_5/pid_accuracy": 0.6390288472175598, "metrics/particle/class_6/efficiency": 0.8658069968223572, "metrics/particle/class_6/pid_accuracy": 0.062029458582401276, "metrics/particle/class_7/efficiency": 0.9690230488777161, "metrics/particle/class_7/pid_accuracy": 0.25573769211769104, "metrics/particle/count/bias_mean": 62.57258987426758, "metrics/particle/count/mae": 85.0478515625, "metrics/particle/count/prediction_mean": 1591.1761474609375, "metrics/particle/count/target_mean": 1528.603515625, "metrics/particle/event/energy_relative_abs_error": 0.08366554230451584, "metrics/particle/event/energy_response_mean": 0.9393139481544495, "metrics/particle/event/met_abs_error": 18.47329330444336, "metrics/particle/event/scalar_pt_relative_abs_error": 0.045392751693725586, "metrics/particle/event/scalar_pt_response_mean": 1.002433180809021, "metrics/particle/event/vector_pt_closure": 0.026978710666298866, "metrics/particle/matched/delta_eta_abs_mean": 0.018739646300673485, "metrics/particle/matched/delta_phi_abs_mean": 0.015743251889944077, "metrics/particle/matched/delta_r_mean": 0.027481364086270332, "metrics/particle/matched/energy_relative_abs_mean": 0.11302297562360764, "metrics/particle/matched/pid_accuracy": 0.8481190204620361, "metrics/particle/matched/pt_relative_abs_mean": 0.10394340753555298, "metrics/particle/matching/duplicate_fraction": 0.10858222097158432, "metrics/particle/matching/efficiency": 0.6412373185157776, "metrics/particle/matching/f1": 0.6283761858940125, "metrics/particle/matching/purity": 0.6160207986831665, "metrics/particle/matching/target_coverage_dr0p05": 0.516219973564148, "metrics/particle/matching/target_coverage_dr0p10": 0.70271897315979, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5617751479148865, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7591044902801514, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017081262543797493, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.06692571938037872, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0582524836063385, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.002278874861076474, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.11095522344112396, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0020820079371333122, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.2186216115951538, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.008446915075182915, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.3221548795700073, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 69164.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4679586887359619, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1459823846817017, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.37786293029785156, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.2537834048271179, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13004735112190247, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 2.6203362941741943, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03993382304906845, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.818933129310608, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.11516214907169342, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 9.352575302124023, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 87645.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7890839576721191, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 28.47577476501465, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 40.96411895751953, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 15.726747512817383, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.9344733357429504, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.5271490812301636, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 35.82422637939453, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -3.5853285789489746, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 53.103118896484375, "metrics/particle/momentum/abs_eta_1to2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 87651.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04041942209005356, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15932120382785797, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15672221779823303, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.015438839793205261, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21060244739055634, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.17302189767360687, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 5.464878559112549, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.01891215145587921, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 11.603103637695312, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 121632.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.5131769180297852, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9782182574272156, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 0.9335900545120239, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 0.7193378210067749, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3177623748779297, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3.527585506439209, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.378118097782135, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 15.713813781738281, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.027110647410154343, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 32.0130500793457, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 228545.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5258374810218811, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 194.91493225097656, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 372.21435546875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 19.30655288696289, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.09172332286834717, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -1.9389430284500122, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 370.07159423828125, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -9.460403442382812, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 556.977294921875, "metrics/particle/momentum/abs_eta_ge2/event/samples": 256.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 231311.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01289394125342369, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.07184720784425735, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.04898051917552948, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01003261562436819, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.10639940202236176, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007660537958145142, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.0775383859872818, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.004186773207038641, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7926270961761475, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 60048.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.33037668466567993, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.2321441173553467, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.25101399421691895, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.15612496435642242, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10131175816059113, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 2.494814872741699, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.022661477327346802, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0633386373519897, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.02906646765768528, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.580824375152588, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 72193.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8317127823829651, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -11.726001739501953, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 18.775054931640625, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 14.284980773925781, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 3.1772701740264893, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.4845153093338013, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 10.130329132080078, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.8057941198348999, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 17.919418334960938, "metrics/particle/momentum/abs_eta_lt1/event/samples": 256.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 72198.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.027395348995923996, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1129055917263031, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10378001630306244, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.004456165246665478, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16616573929786682, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.08465420454740524, "metrics/particle/momentum/all/accepted/pz_abs_mean": 2.7287113666534424, "metrics/particle/momentum/all/accepted/pz_bias": 0.01049711275845766, "metrics/particle/momentum/all/accepted/pz_residual_rms": 8.118758201599121, "metrics/particle/momentum/all/accepted/samples": 250844.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 1.0574462413787842, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.063277006149292, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6813032627105713, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5095873475074768, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.23516735434532166, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 3.1668639183044434, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.2357286512851715, "metrics/particle/momentum/all/assigned/pz_abs_mean": 9.854961395263672, "metrics/particle/momentum/all/assigned/pz_bias": -0.04734446480870247, "metrics/particle/momentum/all/assigned/pz_residual_rms": 25.116851806640625, "metrics/particle/momentum/all/assigned/samples": 388383.0, "metrics/particle/momentum/all/efficiency": 0.641282320022583, "metrics/particle/momentum/all/event/energy_bias": 211.66468811035156, "metrics/particle/momentum/all/event/p3_error_mean": 374.0284423828125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 26.023056030273438, "metrics/particle/momentum/all/event/px_bias": 4.203466892242432, "metrics/particle/momentum/all/event/py_bias": -1.8963091373443604, "metrics/particle/momentum/all/event/pz_abs_mean": 370.1873779296875, "metrics/particle/momentum/all/event/pz_bias": -13.851531982421875, "metrics/particle/momentum/all/event/pz_residual_rms": 555.871826171875, "metrics/particle/momentum/all/event/samples": 256.0, "metrics/particle/momentum/all/target_samples": 391160.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.013643033802509308, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06154054403305054, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05053449794650078, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0072639030404388905, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09499809890985489, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012393859215080738, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.19779638946056366, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.001265968894585967, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.0507807731628418, "metrics/particle/momentum/class_1/accepted/samples": 173473.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.2175755649805069, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.8392917513847351, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.30095112323760986, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2446042001247406, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06864016503095627, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.6485848426818848, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.02261032909154892, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.122408390045166, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.020534750074148178, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.645559310913086, "metrics/particle/momentum/class_1/assigned/samples": 192788.0, "metrics/particle/momentum/class_1/efficiency": 0.8998122215270996, "metrics/particle/momentum/class_1/event/energy_bias": 29.579654693603516, "metrics/particle/momentum/class_1/event/p3_error_mean": 51.83374786376953, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.440631866455078, "metrics/particle/momentum/class_1/event/px_bias": 0.1821097731590271, "metrics/particle/momentum/class_1/event/py_bias": -1.0010980367660522, "metrics/particle/momentum/class_1/event/pz_abs_mean": 46.65244674682617, "metrics/particle/momentum/class_1/event/pz_bias": -5.698270320892334, "metrics/particle/momentum/class_1/event/pz_residual_rms": 79.52983856201172, "metrics/particle/momentum/class_1/event/samples": 256.0, "metrics/particle/momentum/class_1/target_samples": 192788.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.05800675228238106, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.24188782274723053, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23257237672805786, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07257390022277832, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27296409010887146, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.11329362541437149, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 4.26360559463501, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.11987391114234924, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 10.316851615905762, "metrics/particle/momentum/class_2/accepted/samples": 7891.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 2.20046329498291, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.5704759359359741, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.110962152481079, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6223674416542053, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.47233137488365173, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.776092052459717, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.35544031858444214, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 16.7052059173584, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.021373126655817032, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 34.649539947509766, "metrics/particle/momentum/class_2/assigned/samples": 31046.0, "metrics/particle/momentum/class_2/efficiency": 0.25400760769844055, "metrics/particle/momentum/class_2/event/energy_bias": -265.31414794921875, "metrics/particle/momentum/class_2/event/p3_error_mean": 164.3567657470703, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 27.920133590698242, "metrics/particle/momentum/class_2/event/px_bias": 3.0291171073913574, "metrics/particle/momentum/class_2/event/py_bias": -2.0072901248931885, "metrics/particle/momentum/class_2/event/pz_abs_mean": 158.45252990722656, "metrics/particle/momentum/class_2/event/pz_bias": -28.600078582763672, "metrics/particle/momentum/class_2/event/pz_residual_rms": 221.55335998535156, "metrics/particle/momentum/class_2/event/samples": 256.0, "metrics/particle/momentum/class_2/target_samples": 31066.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.06074728071689606, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.22643080353736877, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.2231651246547699, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.007231825962662697, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2627119719982147, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.3154311180114746, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 9.396363258361816, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.007523953914642334, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 14.071189880371094, "metrics/particle/momentum/class_3/accepted/samples": 21003.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 2.024787425994873, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 0.9810692667961121, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.0773324966430664, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.8994393944740295, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.42072567343711853, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 2.7462875843048096, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.5366453528404236, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 20.399620056152344, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03967280685901642, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 33.58567810058594, "metrics/particle/momentum/class_3/assigned/samples": 52476.0, "metrics/particle/momentum/class_3/efficiency": 0.3940451443195343, "metrics/particle/momentum/class_3/event/energy_bias": 540.6463012695312, "metrics/particle/momentum/class_3/event/p3_error_mean": 285.43133544921875, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.533254623413086, "metrics/particle/momentum/class_3/event/px_bias": -0.051150836050510406, "metrics/particle/momentum/class_3/event/py_bias": -1.0669465065002441, "metrics/particle/momentum/class_3/event/pz_abs_mean": 284.96759033203125, "metrics/particle/momentum/class_3/event/pz_bias": -21.273218154907227, "metrics/particle/momentum/class_3/event/pz_residual_rms": 452.9755859375, "metrics/particle/momentum/class_3/event/samples": 256.0, "metrics/particle/momentum/class_3/target_samples": 53301.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.059835903346538544, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.23010019958019257, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22715654969215393, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.0556471087038517, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2663184404373169, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.3598303496837616, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 10.998845100402832, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.05727618560194969, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 17.324155807495117, "metrics/particle/momentum/class_4/accepted/samples": 36784.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 1.8479870557785034, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0703058242797852, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0553972721099854, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.7834035158157349, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.36874696612358093, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.2827820777893066, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.5297839045524597, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 22.249847412109375, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.13346610963344574, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 39.423213958740234, "metrics/particle/momentum/class_4/assigned/samples": 82360.0, "metrics/particle/momentum/class_4/efficiency": 0.4364292025566101, "metrics/particle/momentum/class_4/event/energy_bias": -54.294742584228516, "metrics/particle/momentum/class_4/event/p3_error_mean": 326.7317810058594, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 10.812780380249023, "metrics/particle/momentum/class_4/event/px_bias": -0.5775017738342285, "metrics/particle/momentum/class_4/event/py_bias": -0.5155321955680847, "metrics/particle/momentum/class_4/event/pz_abs_mean": 326.1445007324219, "metrics/particle/momentum/class_4/event/pz_bias": 46.04085922241211, "metrics/particle/momentum/class_4/event/pz_residual_rms": 480.21697998046875, "metrics/particle/momentum/class_4/event/samples": 256.0, "metrics/particle/momentum/class_4/target_samples": 84284.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.05088302120566368, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22434620559215546, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21257014572620392, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03696170076727867, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2531871795654297, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.023625625297427177, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2910304069519043, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.0025661387480795383, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.22584342956543, "metrics/particle/momentum/class_5/accepted/samples": 11005.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 1.4432202577590942, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.1437785625457764, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9692530035972595, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6639377474784851, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.3773454427719116, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 5.050670623779297, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.1498669683933258, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 6.496814250946045, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.16036319732666016, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 17.842121124267578, "metrics/particle/momentum/class_5/assigned/samples": 28939.0, "metrics/particle/momentum/class_5/efficiency": 0.3801775574684143, "metrics/particle/momentum/class_5/event/energy_bias": -29.913997650146484, "metrics/particle/momentum/class_5/event/p3_error_mean": 111.56238555908203, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 24.647005081176758, "metrics/particle/momentum/class_5/event/px_bias": 2.396085739135742, "metrics/particle/momentum/class_5/event/py_bias": 3.345655918121338, "metrics/particle/momentum/class_5/event/pz_abs_mean": 103.65316009521484, "metrics/particle/momentum/class_5/event/pz_bias": -2.9626362323760986, "metrics/particle/momentum/class_5/event/pz_residual_rms": 160.9044952392578, "metrics/particle/momentum/class_5/event/samples": 256.0, "metrics/particle/momentum/class_5/target_samples": 28947.0, "metrics/particle/momentum/class_6/accepted/delta_r_mean": 0.016747113317251205, "metrics/particle/momentum/class_6/accepted/energy_relative_abs_mean": 0.0823223665356636, "metrics/particle/momentum/class_6/accepted/pt_relative_abs_mean": 0.08352576941251755, "metrics/particle/momentum/class_6/accepted/pt_relative_bias": -0.032969098538160324, "metrics/particle/momentum/class_6/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_6/accepted/pt_relative_residual_rms": 0.12358707934617996, "metrics/particle/momentum/class_6/accepted/pz_abs_gt10_fraction": 0.006085192784667015, "metrics/particle/momentum/class_6/accepted/pz_abs_mean": 0.5405703783035278, "metrics/particle/momentum/class_6/accepted/pz_bias": -0.002842262852936983, "metrics/particle/momentum/class_6/accepted/pz_residual_rms": 2.049464702606201, "metrics/particle/momentum/class_6/accepted/samples": 493.0, "metrics/particle/momentum/class_6/assigned/delta_r_mean": 0.3606027066707611, "metrics/particle/momentum/class_6/assigned/energy_relative_abs_mean": 1.0361580848693848, "metrics/particle/momentum/class_6/assigned/pt_relative_abs_mean": 0.997597873210907, "metrics/particle/momentum/class_6/assigned/pt_relative_bias": 0.8550940752029419, "metrics/particle/momentum/class_6/assigned/pt_relative_gt0p5_fraction": 0.11478260904550552, "metrics/particle/momentum/class_6/assigned/pt_relative_residual_rms": 11.677878379821777, "metrics/particle/momentum/class_6/assigned/pz_abs_gt10_fraction": 0.043478261679410934, "metrics/particle/momentum/class_6/assigned/pz_abs_mean": 2.2145159244537354, "metrics/particle/momentum/class_6/assigned/pz_bias": -0.10747987776994705, "metrics/particle/momentum/class_6/assigned/pz_residual_rms": 9.614377975463867, "metrics/particle/momentum/class_6/assigned/samples": 575.0, "metrics/particle/momentum/class_6/efficiency": 0.8573912978172302, "metrics/particle/momentum/class_6/event/energy_bias": -7.774085998535156, "metrics/particle/momentum/class_6/event/p3_error_mean": 6.5965118408203125, "metrics/particle/momentum/class_6/event/pt_vector_error_mean": 2.4249584674835205, "metrics/particle/momentum/class_6/event/px_bias": -0.5981400012969971, "metrics/particle/momentum/class_6/event/py_bias": -0.5442087054252625, "metrics/particle/momentum/class_6/event/pz_abs_mean": 5.714930057525635, "metrics/particle/momentum/class_6/event/pz_bias": -0.9626481533050537, "metrics/particle/momentum/class_6/event/pz_residual_rms": 16.997447967529297, "metrics/particle/momentum/class_6/event/samples": 256.0, "metrics/particle/momentum/class_6/target_samples": 575.0, "metrics/particle/momentum/class_7/accepted/delta_r_mean": 0.012445766478776932, "metrics/particle/momentum/class_7/accepted/energy_relative_abs_mean": 0.04145582765340805, "metrics/particle/momentum/class_7/accepted/pt_relative_abs_mean": 0.03907949477434158, "metrics/particle/momentum/class_7/accepted/pt_relative_bias": 0.0158501248806715, "metrics/particle/momentum/class_7/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_7/accepted/pt_relative_residual_rms": 0.074763223528862, "metrics/particle/momentum/class_7/accepted/pz_abs_gt10_fraction": 0.010256410576403141, "metrics/particle/momentum/class_7/accepted/pz_abs_mean": 0.5992344617843628, "metrics/particle/momentum/class_7/accepted/pz_bias": -0.35300856828689575, "metrics/particle/momentum/class_7/accepted/pz_residual_rms": 2.981494426727295, "metrics/particle/momentum/class_7/accepted/samples": 195.0, "metrics/particle/momentum/class_7/assigned/delta_r_mean": 0.03480742126703262, "metrics/particle/momentum/class_7/assigned/energy_relative_abs_mean": 0.6470197439193726, "metrics/particle/momentum/class_7/assigned/pt_relative_abs_mean": 0.08275464177131653, "metrics/particle/momentum/class_7/assigned/pt_relative_bias": 0.054255302995443344, "metrics/particle/momentum/class_7/assigned/pt_relative_gt0p5_fraction": 0.02010050229728222, "metrics/particle/momentum/class_7/assigned/pt_relative_residual_rms": 0.46206438541412354, "metrics/particle/momentum/class_7/assigned/pz_abs_gt10_fraction": 0.015075377188622952, "metrics/particle/momentum/class_7/assigned/pz_abs_mean": 1.0560451745986938, "metrics/particle/momentum/class_7/assigned/pz_bias": -0.7659808397293091, "metrics/particle/momentum/class_7/assigned/pz_residual_rms": 6.901556491851807, "metrics/particle/momentum/class_7/assigned/samples": 199.0, "metrics/particle/momentum/class_7/efficiency": 0.979899525642395, "metrics/particle/momentum/class_7/event/energy_bias": -1.2643179893493652, "metrics/particle/momentum/class_7/event/p3_error_mean": 2.04351806640625, "metrics/particle/momentum/class_7/event/pt_vector_error_mean": 0.8410587906837463, "metrics/particle/momentum/class_7/event/px_bias": -0.17705292999744415, "metrics/particle/momentum/class_7/event/py_bias": -0.10688969492912292, "metrics/particle/momentum/class_7/event/pz_abs_mean": 1.7119542360305786, "metrics/particle/momentum/class_7/event/pz_bias": -0.39553892612457275, "metrics/particle/momentum/class_7/event/pz_residual_rms": 4.899339199066162, "metrics/particle/momentum/class_7/event/samples": 256.0, "metrics/particle/momentum/class_7/target_samples": 199.0, "metrics/particle/momentum/class_8/accepted/samples": 0.0, "metrics/particle/momentum/class_8/assigned/samples": 0.0, "metrics/particle/momentum/class_8/event/energy_bias": 0.0, "metrics/particle/momentum/class_8/event/p3_error_mean": 0.0, "metrics/particle/momentum/class_8/event/pt_vector_error_mean": 0.0, "metrics/particle/momentum/class_8/event/px_bias": 0.0, "metrics/particle/momentum/class_8/event/py_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_abs_mean": 0.0, "metrics/particle/momentum/class_8/event/pz_bias": 0.0, "metrics/particle/momentum/class_8/event/pz_residual_rms": 0.0, "metrics/particle/momentum/class_8/event/samples": 256.0, "metrics/particle/momentum/class_8/target_samples": 0.0}, "step_train_time": 4.289025783538818, "step_valid_time": 22284.31389594078, "step_total_time": 22288.602921962738, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.021940280073535, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3326253130409902, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81475, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81443, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15703, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 13024, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.1927339674746855, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.1928096951241973, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.1927718238623111, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8071903048758027, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1598527155569193, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1599155237405302, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.15988411348040119, "step/cms_pf_ttbar/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.1927339674746855, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0203454354553763, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.35407520506394075, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 81239, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 81354, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 15237, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 12428, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.1875577001194008, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.18729257319861348, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.18742504289852577, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8127074268013865, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.1529807112347518, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.1527644614892937, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.1528725098866495, "step/cms_pf_qcd/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.1875577001194008, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0182058113521064, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.378156768850603, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 80789, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 80815, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 14464, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 11720, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.17903427446805878, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.17897667512219267, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.17900547016162965, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.8210233248778074, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.14506925447771354, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.14502258244137844, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.14504591470508155, "step/cms_pf_ztt/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.17903427446805878} \ No newline at end of file diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/hyperparameters.json b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..2cbe221f9a66963a43b33ab3c8555bd9ab361543 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 23450849, "dataset": "cms", "data_dir": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/", "model": {"type": "heptv2", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": false, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 3, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "gnnlsh": null, "attention": null, "litept": null, "hept": null, "heptv2": {"activation_checkpointing": false, "embedding_dim": 512, "width": 512, "num_convs": 3, "num_heads": 16, "dropout_ff": 0.0, "activation": "elu", "pe_type": "learned", "block_size": 100, "n_hashes": 3, "num_regions": 140, "deterministic_hashing": true, "num_w_per_dist": 10, "mlp_ratio": 4.0}}, "conv_type": "heptv2", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0004}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 55, "num_classes": 9, "elemtypes_nonzero": [1, 4, 5, 6, 8, 9, 10, 11], "seed": 12345, "num_steps": 100000, "patience": 10000, "checkpoint_freq": 10000, "val_freq": 10000, "num_workers": 2, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 8, "dtype": "bfloat16", "lr": 0.0004, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "cosinedecay", "lr_schedule_config": {}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 100, "validation_diagnostics_batches": 4, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cms": {"physical_pu": {"batch_size": 1, "weight": null, "samples": {"cms_pf_ttbar": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_qcd": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_ztt": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}, "physical_nopu": {"batch_size": 1, "weight": null, "samples": {"cms_pf_ttbar_nopu": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_qcd_nopu": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_ztt_nopu": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cms": {"physical_pu": {"batch_size": 1, "weight": null, "samples": {"cms_pf_ttbar": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_qcd": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_ztt": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}, "physical_nopu": {"batch_size": 1, "weight": null, "samples": {"cms_pf_ttbar_nopu": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_qcd_nopu": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cms_pf_ztt_nopu": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cms_pf_ttbar": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "cms_pf_qcd": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "cms_pf_ztt": {"version": "2.8.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cms_pf_ttbar", "cms_pf_qcd", "cms_pf_ztt"], "ntrain": null, "nvalid": 512, "ntest": 512, "data_config": null, "gpus": 8} \ No newline at end of file diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/model_kwargs.pkl b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a11a60a4c9081757bf9457a74614171a8892f723 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b500c43179b91a0cf45bef512de9644c29970fb96071b9ded9d776fe1bd5d8a3 +size 5769 diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/particleflow_spec.yaml b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..764890baecded6715d37cf35f42c4079f6e00e2c --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/particleflow_spec.yaml @@ -0,0 +1,2551 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.0 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.0 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id019 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id020 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id019 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id020 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_eta.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b7692351daf93d2fc5395f8062b98f572cf0dbe7 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_eta.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_eta.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..f7b48dcef5cc04992c0e36866f65a2f8ccdbc60c Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_eta.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_pt.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..66c92592637589cadb269b82b664a98de6c3f35e Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_pt.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_pt.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..311e8b3f3c1befd9ffc222ef1448f95794e78ea8 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_pt.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_pt_log.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..29ac0c131b84118a86b3ca0c8399f9a597c6c4f1 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_pt_log.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_pt_log.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..41d834c370f75aa3a97a66fa15f11fd054e443d0 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_pt_log.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_res.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e929d2e7056470c728e6e4a5fa7751d863407761 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_res.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_res.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..ee9679157f1fd92078ab391db84e8249e8651ae0 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_res.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_res_vstarget.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..33256900f36851238f85dc43a062ef54de9537cf Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_res_vstarget.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_res_vstarget.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..1aa67ab256463ee7a3b720fd4b26f23a42de6af8 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/jet_res_vstarget.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/num_elements.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..04f15ab32c77a2846701924f5237460485159546 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/num_elements.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/num_elements.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..140b43e3aea253ca07ed2318f9486f8a9f59382f Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_qcd/num_elements.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_eta.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3b9fd317ff61608f26c2a0cd23eac787742e1334 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_eta.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_eta.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..8713508b6bd675bb52c1c0b57753dd8346cc0160 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_eta.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_pt.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a320d94bbc13f15c0a61c8bd663cea54c0faaacd Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_pt.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_pt.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..70c4a644ce92832af614bde2197e75238f912373 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_pt.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_pt_log.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a08fc8460643037003df6a61a992cac786e6fc24 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_pt_log.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_pt_log.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..175f029b42d0919c220e08a84e51d73577910b63 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_pt_log.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_res.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..004e5ab4d60c106bd4c870c7080e45dfc728dbb1 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_res.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_res.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..2971a26d8e113baa9253e5db5aa6468aff3723f8 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_res.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_res_vstarget.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6ae7957b1dfce57e5c8d3826e32cb984983b1a62 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_res_vstarget.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_res_vstarget.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..82e37a70468f8c4b43db0d4b28e279d7688bf730 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/jet_res_vstarget.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/num_elements.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7fc72053953bdfa1c0a2234c8befdfc17509d73a Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/num_elements.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/num_elements.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..6ceb538c6430afebc3faddf55571a5d66c75ae1f Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ttbar/num_elements.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_eta.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..59c919257fd9fb991e9ed7aed7b4ada77b201fb4 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_eta.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_eta.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..897de56d69a3d6dba4e11298bb1d4c158989a1a2 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_eta.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_pt.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9aa86b2eca44ae8fc7810a436a33424d107f5439 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_pt.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_pt.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..d4ad6f3734b36da9a4f078744629bd03a64d62f7 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_pt.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_pt_log.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..10ef21f6fe8ace5bc23c697481c8b68528e599a5 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_pt_log.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_pt_log.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..ef6ed857da7b40ac4be45acee6187e3b992e9f8f Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_pt_log.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_res.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..19b5d398e2e5683dacb82d5300f20832808e4d00 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_res.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_res.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..d35bcfd73d2981a9fa176a32462022b28c3065e0 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_res.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_res_vstarget.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6483b200c858e746ccbef70b2dd2271ab90675b6 Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_res_vstarget.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_res_vstarget.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..5c064df0c279bea17056c41d0e0d497b1ace0a7c Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/jet_res_vstarget.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/num_elements.pdf b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d14861530907d5b0069bda2a3f38fb96d7f4405f Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/num_elements.pdf differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/num_elements.png b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..7f04e088562285379e5949e64677b79cdc285d1f Binary files /dev/null and b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/plots_step_100000/cms_pf_ztt/num_elements.png differ diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/runs/train/events.out.tfevents.1793110276.nid007566.71785.0 b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/runs/train/events.out.tfevents.1793110276.nid007566.71785.0 new file mode 100644 index 0000000000000000000000000000000000000000..b5e0b2b1b92c025822e644c80b63944e9b28a23f --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/runs/train/events.out.tfevents.1793110276.nid007566.71785.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8353e4186f78c9f8726cdad6cfe937dbee2ee94235533c2a4f955ee8692804f8 +size 8962758 diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/runs/valid/events.out.tfevents.1793110276.nid007566.71785.1 b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/runs/valid/events.out.tfevents.1793110276.nid007566.71785.1 new file mode 100644 index 0000000000000000000000000000000000000000..485b8382a229177c443899f00ee8df0592906334 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/runs/valid/events.out.tfevents.1793110276.nid007566.71785.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b72538eb93961c0cf31da579119adc1006df77b37f64f69f167e9c84f9ef70e +size 15843694 diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/scenario-manifest.json b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..b61058c0d9de7ba74d8f9bdb9631a560960327bc --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/scenario-manifest.json @@ -0,0 +1,737 @@ +{ + "scenario": { + "name": "cms_backbone_comparison", + "spec_file": "particleflow_spec.yaml", + "production_name": "cms_run3", + "variants": { + "attention": { + "model_name": "pyg-cms-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.attention.num_heads": 16, + "model.attention.head_dim": 32 + } + }, + "heptv2": { + "model_name": "pyg-cms-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 512, + "model.heptv2.width": 512, + "model.heptv2.num_convs": 3, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 100, + "model.heptv2.n_hashes": 3, + "model.heptv2.deterministic_hashing": true + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 64, + "parameters": { + "lr": 0.0004, + "num_steps": 100000, + "val_freq": 10000, + "checkpoint_freq": 10000, + "tensorboard_step_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "pad_to_multiple_elements": 100, + "make_plots": true + } + }, + "common_overrides": { + "model.output_mode": "elementwise", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 3, + "train_dataset.cms.physical_nopu.batch_size": 1, + "valid_dataset.cms.physical_nopu.batch_size": 1, + "train_dataset.cms.physical_pu.samples.cms_pf_ttbar.version": "2.8.0", + "train_dataset.cms.physical_pu.samples.cms_pf_qcd.version": "2.8.0", + "train_dataset.cms.physical_pu.samples.cms_pf_ztt.version": "2.8.0", + "train_dataset.cms.physical_nopu.samples.cms_pf_ttbar_nopu.version": "2.8.0", + "train_dataset.cms.physical_nopu.samples.cms_pf_qcd_nopu.version": "2.8.0", + "train_dataset.cms.physical_nopu.samples.cms_pf_ztt_nopu.version": "2.8.0", + "valid_dataset.cms.physical_pu.samples.cms_pf_ttbar.version": "2.8.0", + "valid_dataset.cms.physical_pu.samples.cms_pf_qcd.version": "2.8.0", + "valid_dataset.cms.physical_pu.samples.cms_pf_ztt.version": "2.8.0", + "valid_dataset.cms.physical_nopu.samples.cms_pf_ttbar_nopu.version": "2.8.0", + "valid_dataset.cms.physical_nopu.samples.cms_pf_qcd_nopu.version": "2.8.0", + "valid_dataset.cms.physical_nopu.samples.cms_pf_ztt_nopu.version": "2.8.0", + "test_dataset.cms_pf_ttbar.version": "2.8.0", + "test_dataset.cms_pf_qcd.version": "2.8.0", + "test_dataset.cms_pf_ztt.version": "2.8.0" + }, + "allowed_variant_differences": [ + "conv_type", + "model.type", + "model.attention", + "model.heptv2" + ] + }, + "platform": { + "name": "lumi_mi250x", + "gpus": 8, + "python_command": [ + "python3" + ], + "data_dir": { + "cld": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "cms_run3": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/" + }, + "experiments_dir": "/scratch/project_465001293/patajoos/particleflow/experiments", + "environment": { + "MIOPEN_USER_DB_PATH": "/tmp/patajoos-23208735-miopen-cache", + "MIOPEN_CUSTOM_CACHE_DIR": "/tmp/patajoos-23208735-miopen-cache", + "ROCM_PATH": "/opt/rocm", + "KERAS_BACKEND": "torch", + "OMP_NUM_THREADS": "4", + "PYTHONUNBUFFERED": "1", + "NCCL_SOCKET_IFNAME": "hsn", + "NCCL_NET_GDR_LEVEL": "3" + }, + "runtime_overrides": { + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true + }, + "slurm": { + "partition": "standard-g", + "constraint": null, + "account": "project_465001293", + "time": "2-00:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 32, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "480G", + "memory_per_gpu": null, + "no_requeue": true + } + }, + "job": { + "scenario_name": "cms_backbone_comparison", + "platform_name": "lumi_mi250x", + "variant_name": "heptv2", + "model_name": "pyg-cms-v1", + "production_name": "cms_run3", + "data_dir": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/", + "seed": 12345, + "global_batch_size": 64, + "per_gpu_batch_size": 8, + "gpu_batch_multiplier": 8, + "settings": { + "lr": 0.0004, + "num_steps": 100000, + "val_freq": 10000, + "checkpoint_freq": 10000, + "tensorboard_step_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "pad_to_multiple_elements": 100, + "make_plots": true, + "model.output_mode": "elementwise", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 3, + "train_dataset.cms.physical_nopu.batch_size": 1, + "valid_dataset.cms.physical_nopu.batch_size": 1, + "train_dataset.cms.physical_pu.samples.cms_pf_ttbar.version": "2.8.0", + "train_dataset.cms.physical_pu.samples.cms_pf_qcd.version": "2.8.0", + "train_dataset.cms.physical_pu.samples.cms_pf_ztt.version": "2.8.0", + "train_dataset.cms.physical_nopu.samples.cms_pf_ttbar_nopu.version": "2.8.0", + "train_dataset.cms.physical_nopu.samples.cms_pf_qcd_nopu.version": "2.8.0", + "train_dataset.cms.physical_nopu.samples.cms_pf_ztt_nopu.version": "2.8.0", + "valid_dataset.cms.physical_pu.samples.cms_pf_ttbar.version": "2.8.0", + "valid_dataset.cms.physical_pu.samples.cms_pf_qcd.version": "2.8.0", + "valid_dataset.cms.physical_pu.samples.cms_pf_ztt.version": "2.8.0", + "valid_dataset.cms.physical_nopu.samples.cms_pf_ttbar_nopu.version": "2.8.0", + "valid_dataset.cms.physical_nopu.samples.cms_pf_qcd_nopu.version": "2.8.0", + "valid_dataset.cms.physical_nopu.samples.cms_pf_ztt_nopu.version": "2.8.0", + "test_dataset.cms_pf_ttbar.version": "2.8.0", + "test_dataset.cms_pf_qcd.version": "2.8.0", + "test_dataset.cms_pf_ztt.version": "2.8.0", + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true, + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 512, + "model.heptv2.width": 512, + "model.heptv2.num_convs": 3, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 100, + "model.heptv2.n_hashes": 3, + "model.heptv2.deterministic_hashing": true, + "seed": 12345, + "gpu_batch_multiplier": 8 + } + }, + "resolved_config": { + "dataset": "cms", + "data_dir": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/", + "model": { + "type": "heptv2", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": false, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 3, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "gnnlsh": null, + "attention": null, + "litept": null, + "hept": null, + "heptv2": { + "activation_checkpointing": false, + "embedding_dim": 512, + "width": 512, + "num_convs": 3, + "num_heads": 16, + "dropout_ff": 0.0, + "activation": "elu", + "pe_type": "learned", + "block_size": 100, + "n_hashes": 3, + "num_regions": 140, + "deterministic_hashing": true, + "num_w_per_dist": 10, + "mlp_ratio": 4.0 + } + }, + "conv_type": "heptv2", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0004 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 55, + "num_classes": 9, + "elemtypes_nonzero": [ + 1, + 4, + 5, + 6, + 8, + 9, + 10, + 11 + ], + "seed": 12345, + "num_steps": 100000, + "patience": 10000, + "checkpoint_freq": 10000, + "val_freq": 10000, + "num_workers": 2, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 8, + "dtype": "bfloat16", + "lr": 0.0004, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "cosinedecay", + "lr_schedule_config": {}, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 100, + "validation_diagnostics_batches": 4, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cms": { + "physical_pu": { + "batch_size": 1, + "weight": null, + "samples": { + "cms_pf_ttbar": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_qcd": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_ztt": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + }, + "physical_nopu": { + "batch_size": 1, + "weight": null, + "samples": { + "cms_pf_ttbar_nopu": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_qcd_nopu": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_ztt_nopu": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cms": { + "physical_pu": { + "batch_size": 1, + "weight": null, + "samples": { + "cms_pf_ttbar": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_qcd": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_ztt": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + }, + "physical_nopu": { + "batch_size": 1, + "weight": null, + "samples": { + "cms_pf_ttbar_nopu": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_qcd_nopu": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cms_pf_ztt_nopu": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cms_pf_ttbar": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "cms_pf_qcd": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "cms_pf_ztt": { + "version": "2.8.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cms_pf_ttbar", + "cms_pf_qcd", + "cms_pf_ztt" + ], + "ntrain": null, + "nvalid": 512, + "ntest": 512, + "data_config": null, + "gpus": 8 + }, + "command": [ + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cms-v1", + "--production-name", + "cms_run3", + "--data-dir", + "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/", + "--experiment-dir", + "/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470", + "train", + "--gpus", + "8", + "--dtype", + "bfloat16", + "--make-plots", + "--lr", + "0.0004", + "--num_steps", + "100000", + "--val_freq", + "10000", + "--checkpoint_freq", + "10000", + "--tensorboard_step_freq", + "1000", + "--nvalid", + "512", + "--ntest", + "512", + "--sampler_mode", + "interleaved-shards", + "--validation_diagnostics_batches", + "4", + "--pad_to_multiple_elements", + "100", + "--model.output_mode", + "elementwise", + "--model.task_queries", + "false", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "3", + "--train_dataset.cms.physical_nopu.batch_size", + "1", + "--valid_dataset.cms.physical_nopu.batch_size", + "1", + "--train_dataset.cms.physical_pu.samples.cms_pf_ttbar.version", + "2.8.0", + "--train_dataset.cms.physical_pu.samples.cms_pf_qcd.version", + "2.8.0", + "--train_dataset.cms.physical_pu.samples.cms_pf_ztt.version", + "2.8.0", + "--train_dataset.cms.physical_nopu.samples.cms_pf_ttbar_nopu.version", + "2.8.0", + "--train_dataset.cms.physical_nopu.samples.cms_pf_qcd_nopu.version", + "2.8.0", + "--train_dataset.cms.physical_nopu.samples.cms_pf_ztt_nopu.version", + "2.8.0", + "--valid_dataset.cms.physical_pu.samples.cms_pf_ttbar.version", + "2.8.0", + "--valid_dataset.cms.physical_pu.samples.cms_pf_qcd.version", + "2.8.0", + "--valid_dataset.cms.physical_pu.samples.cms_pf_ztt.version", + "2.8.0", + "--valid_dataset.cms.physical_nopu.samples.cms_pf_ttbar_nopu.version", + "2.8.0", + "--valid_dataset.cms.physical_nopu.samples.cms_pf_qcd_nopu.version", + "2.8.0", + "--valid_dataset.cms.physical_nopu.samples.cms_pf_ztt_nopu.version", + "2.8.0", + "--test_dataset.cms_pf_ttbar.version", + "2.8.0", + "--test_dataset.cms_pf_qcd.version", + "2.8.0", + "--test_dataset.cms_pf_ztt.version", + "2.8.0", + "--num_workers", + "2", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "true", + "--model.type", + "heptv2", + "--model.attention", + "null", + "--model.heptv2.embedding_dim", + "512", + "--model.heptv2.width", + "512", + "--model.heptv2.num_convs", + "3", + "--model.heptv2.num_heads", + "16", + "--model.heptv2.dropout_ff", + "0.0", + "--model.heptv2.block_size", + "100", + "--model.heptv2.n_hashes", + "3", + "--model.heptv2.deterministic_hashing", + "true", + "--seed", + "12345", + "--gpu_batch_multiplier", + "8" + ], + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc" +} \ No newline at end of file diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train-config.yaml b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..113f798bf12602b5b7259f90bb1edbe20416fa29 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train-config.yaml @@ -0,0 +1,374 @@ +backend: pytorch +batch_size: 1 +checkpoint_freq: 10000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: heptv2 +data_config: null +data_dir: /scratch/project_465001293/joosep/mlpf/cms/2.8.0/ +dataset: cms +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 4 +- 5 +- 6 +- 8 +- 9 +- 10 +- 11 +enabled_test_datasets: +- cms_pf_ttbar +- cms_pf_qcd +- cms_pf_ztt +gpu_batch_multiplier: 8 +gpu_type: l40 +gpus: 8 +hyperparameters: + batch_size: 1 + lr: 0.0004 +input_dim: 55 +load: null +load_weights_only: false +lr: 0.0004 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: cosinedecay +lr_schedule_config: {} +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: null + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 3 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + gnnlsh: null + hept: null + heptv2: + activation: elu + activation_checkpointing: false + block_size: 100 + deterministic_hashing: true + dropout_ff: 0.0 + embedding_dim: 512 + mlp_ratio: 4.0 + n_hashes: 3 + num_convs: 3 + num_heads: 16 + num_regions: 140 + num_w_per_dist: 10 + pe_type: learned + width: 512 + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: false + trainable: all + type: heptv2 +ntest: 512 +ntrain: null +num_classes: 9 +num_steps: 100000 +num_workers: 2 +nvalid: 512 +optimizer: lamb +pad_to_multiple_elements: 100 +patience: 10000 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cms_pf_qcd: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ttbar: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ztt: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 +threads: 16 +train: true +train_dataset: + cms: + physical_nopu: + batch_size: 1 + samples: + cms_pf_qcd_nopu: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ttbar_nopu: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ztt_nopu: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + weight: null + physical_pu: + batch_size: 1 + samples: + cms_pf_qcd: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ttbar: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ztt: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + weight: null +val_freq: 10000 +valid_dataset: + cms: + physical_nopu: + batch_size: 1 + samples: + cms_pf_qcd_nopu: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ttbar_nopu: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ztt_nopu: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + weight: null + physical_pu: + batch_size: 1 + samples: + cms_pf_qcd: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ttbar: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + cms_pf_ztt: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 2.8.0 + weight: null +validation_diagnostics_batches: 4 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5f1e474c1552c0d0400373c6dd3e4b21252b12d3 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log @@ -0,0 +1,10 @@ +rank0 [2026-10-27 16:10:02,912] INFO : Will use torch.nn.parallel.DistributedDataParallel() and 8 gpus +rank0 [2026-10-27 16:10:03,108] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,108] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,108] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,108] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,108] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,109] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,109] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,109] INFO : AMD Instinct MI250X +rank0 [2026-10-27 16:10:03,109] INFO : Spawning DDP processes diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.0 b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..0b9c0d0e0c7de2008a87e51c4ef52c9312741e32 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.0 @@ -0,0 +1,3821 @@ +rank0 [2026-10-27 16:10:10,938] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-27 16:10:10,939] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-27 16:10:35,267] INFO : Instantiating model +rank0 [2026-10-27 16:10:35,268] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank0 [2026-10-27 16:10:35,268] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank0 [2026-10-27 16:10:35,296] INFO : Input encoding initialization took 0.03s +rank0 [2026-10-27 16:10:35,296] INFO : nn0 parameters: 2130944 +rank0 [2026-10-27 16:10:35,296] INFO : Initializing shared backbone layers, num_convs=3 +rank0 [2026-10-27 16:10:35,296] INFO : Initializing backbone layer 0 +rank0 [2026-10-27 16:10:35,350] INFO : Initializing backbone layer 1 +rank0 [2026-10-27 16:10:35,379] INFO : Initializing backbone layer 2 +rank0 [2026-10-27 16:10:35,408] INFO : Backbone initialization took 0.11s +rank0 [2026-10-27 16:10:35,409] INFO : backbone parameters: 15765696 +rank0 [2026-10-27 16:10:35,409] INFO : Initializing output DNNs +rank0 [2026-10-27 16:10:35,442] INFO : Output DNNs initialization took 0.03s +rank0 [2026-10-27 16:10:35,442] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-27 16:10:35,442] INFO : jagged_attention enabled=False +rank0 [2026-10-27 16:10:35,442] INFO : flash_attn_varlen enabled=False +rank0 [2026-10-27 16:10:35,442] INFO : task_queries enabled=False +rank0 [2026-10-27 16:10:35,442] INFO : classification_norm parameters: 0 +rank0 [2026-10-27 16:10:35,443] INFO : regression_norm parameters: 0 +rank0 [2026-10-27 16:10:35,443] INFO : classification_query parameters: 0 +rank0 [2026-10-27 16:10:35,443] INFO : regression_query parameters: 0 +rank0 [2026-10-27 16:10:35,443] INFO : classification_readout parameters: 0 +rank0 [2026-10-27 16:10:35,443] INFO : regression_readout parameters: 0 +rank0 [2026-10-27 16:10:35,443] INFO : output_mode=elementwise +rank0 [2026-10-27 16:10:35,443] INFO : set_decoder parameters: 0 +rank0 [2026-10-27 16:10:35,443] INFO : nn_binary_particle parameters: 264706 +rank0 [2026-10-27 16:10:35,443] INFO : nn_pid parameters: 268297 +rank0 [2026-10-27 16:10:35,443] INFO : nn_pt parameters: 2113544 +rank0 [2026-10-27 16:10:35,443] INFO : nn_eta parameters: 264706 +rank0 [2026-10-27 16:10:35,443] INFO : nn_sin_phi parameters: 264706 +rank0 [2026-10-27 16:10:35,443] INFO : nn_cos_phi parameters: 264706 +rank0 [2026-10-27 16:10:35,443] INFO : nn_energy parameters: 2113544 +rank0 [2026-10-27 16:10:35,444] INFO : Total MLPF parameters: 23450849 +rank0 [2026-10-27 16:10:35,444] INFO : MLPF __init__ done +rank0 [2026-10-27 16:10:35,444] INFO : Instantiated model +rank0 [2026-10-27 16:10:35,444] INFO : Moving model to device rank=0 +rank0 [2026-10-27 16:10:35,475] INFO : Moved model to device rank=0 +rank0 [2026-10-27 16:10:35,476] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-27 16:10:35,477] INFO : Configured model for SyncBatchNorm rank=0 +rank0 [2026-10-27 16:10:36,005] INFO : Configured model for DistributedDataParallel rank=0 +rank0 [2026-10-27 16:10:36,125] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +202 module.nn_energy.nn.7.0.bias ... 0 +203 module.nn_energy.nn.7.2.weight ... 0 +204 module.nn_energy.nn.7.2.bias ... 0 +205 module.nn_energy.nn.7.4.weight ... 0 +206 module.nn_energy.nn.7.4.bias ... 0 + +[207 rows x 3 columns] +rank0 [2026-10-27 16:10:36,125] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x HEPTv2Layer( + embedding_dim=512, num_heads=16, width=512, dropout=0.0, block_size=100, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=512, out_features=512, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=512, out_features=512, bias=False) + (w_k): Linear(in_features=512, out_features=512, bias=False) + (w_v): Linear(in_features=512, out_features=512, bias=False) + (w_rpe): Linear(in_features=10, out_features=512, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=2048, bias=True) + (1): LayerNorm((2048,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=2048, out_features=512, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=512, out_features=2048, bias=False) + (up_proj): Linear(in_features=512, out_features=2048, bias=False) + (down_proj): Linear(in_features=2048, out_features=512, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank0 [2026-10-27 16:10:36,126] INFO : Trainable parameters: 23450849 +rank0 [2026-10-27 16:10:36,126] INFO : Non-trainable parameters: 22386 +rank0 [2026-10-27 16:10:36,126] INFO : Total parameters: 23473235 +rank0 [2026-10-27 16:10:36,129] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 262144 0 + module.backbone.0.attn.out_linear.bias 512 0 + module.backbone.0.attn.e2lsh.alpha 0 1632 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 262144 0 + module.backbone.0.w_k.weight 262144 0 + module.backbone.0.w_v.weight 262144 0 + module.backbone.0.w_rpe.weight 0 5120 + module.backbone.0.w_rpe.bias 0 512 +module.backbone.0.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm2.weight 512 0 + module.backbone.0.q_norm.weight 32 0 + module.backbone.0.k_norm.weight 32 0 + module.backbone.0.ff.gate_proj.weight 1048576 0 + module.backbone.0.ff.up_proj.weight 1048576 0 + module.backbone.0.ff.down_proj.weight 1048576 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 262144 0 + module.backbone.1.attn.out_linear.bias 512 0 + module.backbone.1.attn.e2lsh.alpha 0 1632 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 262144 0 + module.backbone.1.w_k.weight 262144 0 + module.backbone.1.w_v.weight 262144 0 + module.backbone.1.w_rpe.weight 0 5120 + module.backbone.1.w_rpe.bias 0 512 +module.backbone.1.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm2.weight 512 0 + module.backbone.1.q_norm.weight 32 0 + module.backbone.1.k_norm.weight 32 0 + module.backbone.1.ff.gate_proj.weight 1048576 0 + module.backbone.1.ff.up_proj.weight 1048576 0 + module.backbone.1.ff.down_proj.weight 1048576 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 262144 0 + module.backbone.2.attn.out_linear.bias 512 0 + module.backbone.2.attn.e2lsh.alpha 0 1632 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 262144 0 + module.backbone.2.w_k.weight 262144 0 + module.backbone.2.w_v.weight 262144 0 + module.backbone.2.w_rpe.weight 0 5120 + module.backbone.2.w_rpe.bias 0 512 +module.backbone.2.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm2.weight 512 0 + module.backbone.2.q_norm.weight 32 0 + module.backbone.2.k_norm.weight 32 0 + module.backbone.2.ff.gate_proj.weight 1048576 0 + module.backbone.2.ff.up_proj.weight 1048576 0 + module.backbone.2.ff.down_proj.weight 1048576 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank0 [2026-10-27 16:10:36,132] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470 +rank0 [2026-10-27 16:10:36,132] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470 +rank0 [2026-10-27 16:10:36,132] INFO : Getting dataloaders +rank0 [2026-10-27 16:10:36,133] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:42,608] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:42,684] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:42,754] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:42,825] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:42,905] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:42,973] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:43,044] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:43,131] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:43,186] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:43,312] INFO : train_dataset: cms_pf_ttbar, 45000 +rank0 [2026-10-27 16:10:43,312] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:43,382] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:43,452] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:43,526] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:43,609] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:43,682] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:43,768] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:44,082] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:44,133] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:44,183] INFO : train_dataset: cms_pf_qcd, 44950 +rank0 [2026-10-27 16:10:44,231] INFO : train_dataset: cms_pf_qcd, 45400 +rank0 [2026-10-27 16:10:44,231] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:44,280] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,328] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,373] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,421] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,479] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,548] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,638] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,691] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,747] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,830] INFO : train_dataset: cms_pf_ztt, 45000 +rank0 [2026-10-27 16:10:44,830] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-27 16:10:44,830] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:44,931] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,034] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,142] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,240] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,328] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,435] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,544] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,636] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:45,976] INFO : train_dataset: cms_pf_ttbar_nopu, 450000 +rank0 [2026-10-27 16:10:46,056] INFO : train_dataset: cms_pf_ttbar_nopu, 450900 +rank0 [2026-10-27 16:10:46,057] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:46,134] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,212] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,295] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,389] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,495] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,599] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,685] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,787] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,903] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,995] INFO : train_dataset: cms_pf_qcd_nopu, 450000 +rank0 [2026-10-27 16:10:46,995] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:47,077] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,133] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,204] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,292] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,359] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,684] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,734] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,785] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,845] INFO : train_dataset: cms_pf_ztt_nopu, 450000 +rank0 [2026-10-27 16:10:47,894] INFO : train_dataset: cms_pf_ztt_nopu, 450900 +rank0 [2026-10-27 16:10:47,895] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-27 16:10:47,895] INFO : Creating WeightedMixIterator with 2 data loaders. +rank0 [2026-10-27 16:10:47,895] INFO : train mix: physical_pu: weight=21093 step_share=0.091 batches_per_epoch=21093 events_per_step=64 expected_passes_in_100000_steps=0.43 +rank0 [2026-10-27 16:10:47,895] INFO : train mix: physical_nopu: weight=210965 step_share=0.909 batches_per_epoch=210965 events_per_step=64 expected_passes_in_100000_steps=0.43 +rank0 [2026-10-27 16:10:47,895] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:47,910] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,925] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,941] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,956] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,970] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:47,986] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:48,001] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:48,017] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:48,032] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:48,049] INFO : valid_dataset: cms_pf_ttbar, 51 +rank0 [2026-10-27 16:10:48,049] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:48,065] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,081] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,096] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,111] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,127] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,144] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,158] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,173] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,187] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,202] INFO : valid_dataset: cms_pf_qcd, 51 +rank0 [2026-10-27 16:10:48,202] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:48,217] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,232] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,247] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,262] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,277] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,292] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,306] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,321] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,336] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,351] INFO : valid_dataset: cms_pf_ztt, 51 +rank0 [2026-10-27 16:10:48,351] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-27 16:10:48,351] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:48,369] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,385] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,402] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,419] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,435] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,452] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,468] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,485] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,502] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,519] INFO : valid_dataset: cms_pf_ttbar_nopu, 51 +rank0 [2026-10-27 16:10:48,519] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:48,536] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,553] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,570] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,587] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,605] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,622] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,639] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,656] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,674] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,690] INFO : valid_dataset: cms_pf_qcd_nopu, 51 +rank0 [2026-10-27 16:10:48,691] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:10:48,706] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,721] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,736] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,751] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,768] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,781] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,795] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,809] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,824] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,837] INFO : valid_dataset: cms_pf_ztt_nopu, 51 +rank0 [2026-10-27 16:10:48,838] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-27 16:10:48,838] INFO : Creating InterleavedIterator with 2 data loaders. +rank0 [2026-10-27 16:11:16,593] INFO : Got dataloaders +rank0 [2026-10-27 16:11:16,593] INFO : loader split=train rank=0 len=232058 +rank0 [2026-10-27 16:11:16,593] INFO : loader split=valid rank=0 len=48 +rank0 [2026-10-27 16:11:16,594] INFO : Starting training from step 1 to 100000 on rank 0 of 8 +rank0 [2026-10-27 16:11:16,599] INFO : Creating train_iterator +rank0 [2026-10-27 16:11:16,599] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank0 [2026-10-27 16:11:30,716] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank0 [2026-10-27 16:11:46,215] INFO : Created train_iterator +rank0 [2026-10-27 16:12:32,258] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.129867712855339, 'Classification': 0.16580097352620216, 'Regression_pt': 0.7168090478330851, 'Regression_eta': 0.335200086296536, 'Regression_sin_phi': 0.1274821354681626, 'Regression_cos_phi': 0.1371471309568733, 'Regression_energy': 0.9554258581250906}, weights={'Classification_binary': 1.0, 'Classification': 30.939912796020508, 'Regression_pt': 7.156533241271973, 'Regression_eta': 15.3038969039917, 'Regression_sin_phi': 40.23989486694336, 'Regression_cos_phi': 37.40412139892578, 'Regression_energy': 5.369194984436035} +rank0 [2026-10-27 16:15:18,888] INFO : Step 1000/100000 rank0 | Train Loss: 16.8632 | Train Loss Unweighted: 5.4934 | Train Loss Components: Classification: 0.0189 | Classification_binary: 4.1833 | Regression_cos_phi: 0.0509 | Regression_energy: 0.5806 | Regression_eta: 0.0776 | Regression_pt: 0.5302 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 5.1301 | physical_nopu: 6.0772 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5524 | Classification_binary: 4.2254 | Regression_cos_phi: 2.0949 | Regression_energy: 3.1201 | Regression_eta: 1.4487 | Regression_pt: 3.5694 | Regression_sin_phi: 1.7882 | LR: 4.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.8783s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0014 | max_forward: 0.0387 | max_loss: 0.0036 | max_backward: 0.0664 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 1.2817 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 4374.6 | uss_MB: 3102.3 | child_rss_MB: 11636.1 | allocated_MB: 609.1 | reserved_MB: 27788.0 | inactive_split_MB: 244.1 +rank0 [2026-10-27 16:15:19,146] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 49.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 48.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 25% +6 10 0x7408, 62036 53.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:18:42,691] INFO : Step 2000/100000 rank0 | Train Loss: 17.7204 | Train Loss Unweighted: 5.4242 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.9240 | Regression_cos_phi: 0.0505 | Regression_energy: 0.6758 | Regression_eta: 0.0665 | Regression_pt: 0.6366 | Regression_sin_phi: 0.0551 | Group Train Loss Unweighted (window mean): physical_pu: 4.2802 | physical_nopu: 5.3470 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4688 | Classification_binary: 3.9663 | Regression_cos_phi: 1.5001 | Regression_energy: 3.4131 | Regression_eta: 0.8606 | Regression_pt: 4.3202 | Regression_sin_phi: 1.8125 | LR: 4.00e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.4871s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1422 | max_loss: 0.0036 | max_backward: 0.1717 | max_optimizer_step: 0.1501 | max_tensorboard_logging: 0.7287 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 4367.5 | uss_MB: 3102.8 | child_rss_MB: 11637.6 | allocated_MB: 607.0 | reserved_MB: 27788.0 | inactive_split_MB: 326.9 +rank0 [2026-10-27 16:18:42,944] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 49.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 49.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 52.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:22:11,163] INFO : Step 3000/100000 rank0 | Train Loss: 14.3102 | Train Loss Unweighted: 4.8875 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.8970 | Regression_cos_phi: 0.0491 | Regression_energy: 0.4277 | Regression_eta: 0.0716 | Regression_pt: 0.3769 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 4.0705 | physical_nopu: 5.1319 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3406 | Classification_binary: 3.9444 | Regression_cos_phi: 1.5742 | Regression_energy: 2.1989 | Regression_eta: 0.9446 | Regression_pt: 2.5579 | Regression_sin_phi: 1.7900 | LR: 3.99e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.6575s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0782 | max_loss: 0.0037 | max_backward: 0.0505 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 1.0271 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4295.2 | uss_MB: 3034.9 | child_rss_MB: 11625.5 | allocated_MB: 607.6 | reserved_MB: 28576.0 | inactive_split_MB: 261.0 +rank0 [2026-10-27 16:22:11,424] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 49.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 49.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 33% 0% +6 10 0x7408, 62036 52.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:25:44,884] INFO : Step 4000/100000 rank0 | Train Loss: 12.6160 | Train Loss Unweighted: 3.9080 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.1145 | Regression_cos_phi: 0.0484 | Regression_energy: 0.3088 | Regression_eta: 0.1117 | Regression_pt: 0.2597 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.9908 | physical_nopu: 5.0219 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4786 | Classification_binary: 3.0744 | Regression_cos_phi: 1.7260 | Regression_energy: 1.5856 | Regression_eta: 1.5528 | Regression_pt: 1.8051 | Regression_sin_phi: 1.8112 | LR: 3.99e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.0872s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0032 | max_forward: 0.1365 | max_loss: 0.0036 | max_backward: 0.2496 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 1.0566 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 4292.4 | uss_MB: 3043.1 | child_rss_MB: 11491.5 | allocated_MB: 617.3 | reserved_MB: 28576.0 | inactive_split_MB: 1049.1 +rank0 [2026-10-27 16:25:45,146] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 50.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 49.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 52.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:29:29,688] INFO : Step 5000/100000 rank0 | Train Loss: 13.7309 | Train Loss Unweighted: 4.7575 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.8762 | Regression_cos_phi: 0.0508 | Regression_energy: 0.3661 | Regression_eta: 0.0922 | Regression_pt: 0.3105 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.9453 | physical_nopu: 4.9561 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4077 | Classification_binary: 3.8939 | Regression_cos_phi: 2.0189 | Regression_energy: 1.9809 | Regression_eta: 1.7788 | Regression_pt: 2.2086 | Regression_sin_phi: 1.7673 | LR: 3.98e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.7463s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.0008 | max_forward: 0.0371 | max_loss: 0.0036 | max_backward: 0.0568 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 2.1174 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4241.6 | uss_MB: 3036.2 | child_rss_MB: 11146.9 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 248.7 +rank0 [2026-10-27 16:29:29,938] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 47.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 51.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:33:04,433] INFO : Step 6000/100000 rank0 | Train Loss: 15.2316 | Train Loss Unweighted: 4.9080 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.7479 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5147 | Regression_eta: 0.0679 | Regression_pt: 0.4609 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.9059 | physical_nopu: 4.9043 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6045 | Classification_binary: 3.9872 | Regression_cos_phi: 2.3503 | Regression_energy: 2.4639 | Regression_eta: 1.1493 | Regression_pt: 2.7230 | Regression_sin_phi: 2.5798 | LR: 3.97e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.6959s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0421 | max_loss: 0.0036 | max_backward: 0.0620 | max_optimizer_step: 0.1489 | max_tensorboard_logging: 2.0592 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4216.9 | uss_MB: 3036.3 | child_rss_MB: 11044.6 | allocated_MB: 610.3 | reserved_MB: 28576.0 | inactive_split_MB: 254.0 +rank0 [2026-10-27 16:33:04,691] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:36:50,536] INFO : Step 7000/100000 rank0 | Train Loss: 14.7215 | Train Loss Unweighted: 4.8917 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8534 | Regression_cos_phi: 0.0503 | Regression_energy: 0.4508 | Regression_eta: 0.0791 | Regression_pt: 0.3921 | Regression_sin_phi: 0.0547 | Group Train Loss Unweighted (window mean): physical_pu: 3.8763 | physical_nopu: 4.8627 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3857 | Classification_binary: 3.6878 | Regression_cos_phi: 1.9107 | Regression_energy: 3.5490 | Regression_eta: 1.4895 | Regression_pt: 4.3352 | Regression_sin_phi: 2.9968 | LR: 3.96e-04 | DataLoad Time: 0.0019s | Model Forward Time: 5.1745s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0348 | max_loss: 0.0036 | max_backward: 0.0535 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 3.5409 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4194.7 | uss_MB: 3033.5 | child_rss_MB: 10983.9 | allocated_MB: 609.0 | reserved_MB: 28576.0 | inactive_split_MB: 216.2 +rank0 [2026-10-27 16:36:50,797] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 48.0°C 123.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 47.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:40:34,837] INFO : Step 8000/100000 rank0 | Train Loss: 14.2560 | Train Loss Unweighted: 4.8042 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.8045 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4333 | Regression_eta: 0.0601 | Regression_pt: 0.3883 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.8518 | physical_nopu: 4.8294 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4507 | Classification_binary: 3.7338 | Regression_cos_phi: 1.8701 | Regression_energy: 2.1700 | Regression_eta: 1.0173 | Regression_pt: 2.6169 | Regression_sin_phi: 2.1140 | LR: 3.94e-04 | DataLoad Time: 0.0019s | Model Forward Time: 5.5804s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1342 | max_loss: 0.0036 | max_backward: 0.0500 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 3.8645 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4176.9 | uss_MB: 3036.4 | child_rss_MB: 10934.3 | allocated_MB: 606.6 | reserved_MB: 28576.0 | inactive_split_MB: 255.4 +rank0 [2026-10-27 16:40:35,097] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 48.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 48.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:44:13,988] INFO : Step 9000/100000 rank0 | Train Loss: 15.9274 | Train Loss Unweighted: 4.9361 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.6437 | Regression_cos_phi: 0.0472 | Regression_energy: 0.5701 | Regression_eta: 0.0733 | Regression_pt: 0.5360 | Regression_sin_phi: 0.0496 | Group Train Loss Unweighted (window mean): physical_pu: 3.8376 | physical_nopu: 4.8057 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4512 | Classification_binary: 3.8137 | Regression_cos_phi: 1.4588 | Regression_energy: 2.5921 | Regression_eta: 1.1048 | Regression_pt: 3.1300 | Regression_sin_phi: 1.8845 | LR: 3.93e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.8900s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1320 | max_loss: 0.0037 | max_backward: 0.0555 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 2.0797 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 4157.5 | uss_MB: 3035.9 | child_rss_MB: 10877.2 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 321.4 +rank0 [2026-10-27 16:44:14,238] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +2 6 0x7408, 15961 48.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:48:06,075] INFO : Step 10000/100000 rank0 | Train Loss: 13.0514 | Train Loss Unweighted: 4.5682 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.7678 | Regression_cos_phi: 0.0520 | Regression_energy: 0.3372 | Regression_eta: 0.0736 | Regression_pt: 0.2745 | Regression_sin_phi: 0.0521 | Group Train Loss Unweighted (window mean): physical_pu: 3.8237 | physical_nopu: 4.7797 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3243 | Classification_binary: 3.7735 | Regression_cos_phi: 2.0821 | Regression_energy: 1.6440 | Regression_eta: 1.0792 | Regression_pt: 1.8286 | Regression_sin_phi: 2.0650 | LR: 3.91e-04 | DataLoad Time: 0.0011s | Model Forward Time: 3.8978s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.0379 | max_loss: 0.0040 | max_backward: 0.0556 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 2.2516 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4149.4 | uss_MB: 3036.1 | child_rss_MB: 10872.4 | allocated_MB: 607.9 | reserved_MB: 28576.0 | inactive_split_MB: 258.7 +rank0 [2026-10-27 16:48:06,332] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 49.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 47.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:48:06,484] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/checkpoints/checkpoint-10000.pth +rank0 [2026-10-27 16:48:07,022] INFO : Running validation on rank0 +rank0 [2026-10-27 16:48:07,022] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 16:48:07,022] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 16:48:07,022] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 16:48:07,022] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 16:48:15,337] INFO : saved batch0_epoch10000.parquet +rank0 [2026-10-27 16:48:15,611] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 16:48:15,738] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 16:48:15,808] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 16:48:15,880] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 16:48:15,941] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 16:48:16,015] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 16:48:16,085] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 16:48:16,148] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 16:48:16,246] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 16:48:16,341] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 16:48:16,408] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 16:48:16,478] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 16:48:16,536] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 16:48:16,608] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 16:48:16,679] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 16:48:16,749] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 16:48:16,835] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 16:48:16,921] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 16:48:16,987] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 16:48:17,058] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 16:48:17,114] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 16:48:17,187] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 16:48:17,258] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 16:48:17,320] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 16:48:17,415] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 16:48:17,505] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 16:48:17,572] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 16:48:17,643] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 16:48:17,698] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 16:48:17,766] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 16:48:17,835] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 16:48:17,892] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 16:48:18,540] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 16:48:18,640] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 16:48:18,710] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 16:48:18,784] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 16:48:18,843] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 16:48:18,916] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 16:48:18,989] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 16:48:19,051] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 16:48:19,153] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 16:48:19,255] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 16:48:19,328] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 16:48:19,401] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 16:48:19,463] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 16:48:19,535] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 16:48:19,609] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 16:48:19,672] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 16:48:19,764] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 16:48:19,857] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 16:48:19,926] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 16:48:19,993] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 16:48:20,053] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 16:48:20,123] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 16:48:20,196] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 16:48:20,266] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 16:48:20,364] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 16:48:20,450] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 16:48:20,518] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 16:48:20,589] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 16:48:20,646] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 16:48:20,715] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 16:48:20,784] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 16:48:20,852] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 16:48:20,856] INFO : plotted pt_target histogram +rank0 [2026-10-27 16:48:20,858] INFO : plotted pt_pred histogram +rank0 [2026-10-27 16:48:20,861] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 16:48:20,864] INFO : plotted eta_target histogram +rank0 [2026-10-27 16:48:20,866] INFO : plotted eta_pred histogram +rank0 [2026-10-27 16:48:20,869] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 16:48:20,871] INFO : plotted sphi_target histogram +rank0 [2026-10-27 16:48:20,874] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 16:48:20,877] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 16:48:20,878] INFO : plotted cphi_target histogram +rank0 [2026-10-27 16:48:20,881] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 16:48:20,885] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 16:48:20,887] INFO : plotted energy_target histogram +rank0 [2026-10-27 16:48:20,890] INFO : plotted energy_pred histogram +rank0 [2026-10-27 16:48:20,893] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 16:50:06,796] INFO : VALIDATION | Step=10000/100000 | Train Loss=13.0514 | Valid Loss=13.4312 | Train Loss Unweighted=4.5682 | Valid Loss Unweighted=4.2924 | Group Valid Loss Unweighted: physical_pu: 3.8115 | physical_nopu: 4.7733 | Stale=0 | ETA=349.5m +rank0 [2026-10-27 16:50:06,810] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:50:06,965] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 16:50:06,988] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 16:50:35,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 16:50:35,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 16:50:36,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 16:50:36,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 16:50:37,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 16:50:37,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 16:50:38,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 16:50:38,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 16:50:48,959] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-27 16:50:48,967] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:50:49,335] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 16:50:49,338] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 16:51:04,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 16:51:05,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 16:51:05,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 16:51:06,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 16:51:06,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 16:51:07,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 16:51:07,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 16:51:08,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 16:51:18,458] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 16:51:18,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 16:51:19,153] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 16:51:19,157] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 16:51:34,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 16:51:34,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 16:51:35,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 16:51:35,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 16:51:36,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 16:51:37,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 16:51:37,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 16:51:38,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 16:51:48,167] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 16:51:48,552] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar +rank0 [2026-10-27 16:51:53,820] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar +rank0 [2026-10-27 16:51:55,231] INFO : Plotted number of elements +rank0 [2026-10-27 16:51:56,932] INFO : Plotted jets +rank0 [2026-10-27 16:51:58,122] INFO : Plotted jet ratio +rank0 [2026-10-27 16:51:58,178] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd +rank0 [2026-10-27 16:52:03,906] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd +rank0 [2026-10-27 16:52:05,219] INFO : Plotted number of elements +rank0 [2026-10-27 16:52:06,935] INFO : Plotted jets +rank0 [2026-10-27 16:52:08,107] INFO : Plotted jet ratio +rank0 [2026-10-27 16:52:08,146] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt +rank0 [2026-10-27 16:52:13,409] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt +rank0 [2026-10-27 16:52:14,628] INFO : Plotted number of elements +rank0 [2026-10-27 16:52:16,357] INFO : Plotted jets +rank0 [2026-10-27 16:52:17,547] INFO : Plotted jet ratio +rank0 [2026-10-27 16:55:59,391] INFO : Step 11000/100000 rank0 | Train Loss: 14.4095 | Train Loss Unweighted: 4.7539 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.7591 | Regression_cos_phi: 0.0522 | Regression_energy: 0.4228 | Regression_eta: 0.0709 | Regression_pt: 0.3796 | Regression_sin_phi: 0.0518 | Group Train Loss Unweighted (window mean): physical_pu: 3.8077 | physical_nopu: 4.7444 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4818 | Classification_binary: 3.7267 | Regression_cos_phi: 2.2979 | Regression_energy: 2.0861 | Regression_eta: 1.2144 | Regression_pt: 2.4510 | Regression_sin_phi: 2.1214 | LR: 3.89e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.8883s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1127 | max_loss: 0.0074 | max_backward: 0.1410 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 1.9404 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6580.9 | uss_MB: 5462.7 | child_rss_MB: 10900.1 | allocated_MB: 610.6 | reserved_MB: 27422.0 | inactive_split_MB: 253.4 +rank0 [2026-10-27 16:55:59,652] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 6% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +2 6 0x7408, 15961 49.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 47.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 16:59:47,057] INFO : Step 12000/100000 rank0 | Train Loss: 16.4725 | Train Loss Unweighted: 5.0863 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.5958 | Regression_cos_phi: 0.0410 | Regression_energy: 0.6842 | Regression_eta: 0.0610 | Regression_pt: 0.6484 | Regression_sin_phi: 0.0394 | Group Train Loss Unweighted (window mean): physical_pu: 3.8013 | physical_nopu: 4.7308 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4732 | Classification_binary: 3.6358 | Regression_cos_phi: 2.1434 | Regression_energy: 3.2805 | Regression_eta: 1.0336 | Regression_pt: 3.9078 | Regression_sin_phi: 1.5498 | LR: 3.87e-04 | DataLoad Time: 0.0022s | Model Forward Time: 4.2771s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1292 | max_loss: 0.0037 | max_backward: 0.1951 | max_optimizer_step: 0.2940 | max_tensorboard_logging: 2.1160 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6552.9 | uss_MB: 5462.9 | child_rss_MB: 10800.4 | allocated_MB: 609.5 | reserved_MB: 27422.0 | inactive_split_MB: 347.2 +rank0 [2026-10-27 16:59:47,321] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 49.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 51.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:03:41,211] INFO : Step 13000/100000 rank0 | Train Loss: 12.9993 | Train Loss Unweighted: 4.5752 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.7655 | Regression_cos_phi: 0.0483 | Regression_energy: 0.3375 | Regression_eta: 0.0674 | Regression_pt: 0.2918 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7898 | physical_nopu: 4.7063 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4680 | Classification_binary: 3.7420 | Regression_cos_phi: 1.7868 | Regression_energy: 1.6925 | Regression_eta: 1.1332 | Regression_pt: 1.8360 | Regression_sin_phi: 2.5406 | LR: 3.85e-04 | DataLoad Time: 0.0090s | Model Forward Time: 3.7472s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1327 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.9573 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5436.1 | uss_MB: 4383.7 | child_rss_MB: 10499.2 | allocated_MB: 607.2 | reserved_MB: 28584.0 | inactive_split_MB: 340.8 +rank0 [2026-10-27 17:03:41,468] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:07:24,336] INFO : Step 14000/100000 rank0 | Train Loss: 12.1035 | Train Loss Unweighted: 3.7686 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.0394 | Regression_cos_phi: 0.0492 | Regression_energy: 0.2767 | Regression_eta: 0.1064 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0483 | Group Train Loss Unweighted (window mean): physical_pu: 3.7803 | physical_nopu: 4.6878 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5464 | Classification_binary: 3.0591 | Regression_cos_phi: 1.8367 | Regression_energy: 1.4653 | Regression_eta: 1.5402 | Regression_pt: 1.6533 | Regression_sin_phi: 1.9983 | LR: 3.83e-04 | DataLoad Time: 0.0018s | Model Forward Time: 6.1931s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0034 | max_forward: 0.2002 | max_loss: 0.0036 | max_backward: 0.3668 | max_optimizer_step: 0.2700 | max_tensorboard_logging: 3.9098 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0824 | Memory: rss_MB: 5443.1 | uss_MB: 4393.2 | child_rss_MB: 10497.0 | allocated_MB: 618.5 | reserved_MB: 28584.0 | inactive_split_MB: 893.5 +rank0 [2026-10-27 17:07:24,591] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 3% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +2 6 0x7408, 15961 48.0°C 122.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 48.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 51.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:11:08,535] INFO : Step 15000/100000 rank0 | Train Loss: 13.2483 | Train Loss Unweighted: 4.5172 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.6965 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3441 | Regression_eta: 0.0751 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7702 | physical_nopu: 4.6706 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3216 | Classification_binary: 3.7712 | Regression_cos_phi: 2.1035 | Regression_energy: 1.5049 | Regression_eta: 1.7493 | Regression_pt: 1.5797 | Regression_sin_phi: 2.3605 | LR: 3.80e-04 | DataLoad Time: 0.0015s | Model Forward Time: 3.9611s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0879 | max_loss: 0.0036 | max_backward: 0.0859 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 2.2540 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0190 | Memory: rss_MB: 5437.1 | uss_MB: 4385.0 | child_rss_MB: 10498.2 | allocated_MB: 605.9 | reserved_MB: 28584.0 | inactive_split_MB: 314.1 +rank0 [2026-10-27 17:11:08,790] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 48.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:14:56,534] INFO : Step 16000/100000 rank0 | Train Loss: 14.0060 | Train Loss Unweighted: 4.5724 | Train Loss Components: Classification: 0.0150 | Classification_binary: 3.5827 | Regression_cos_phi: 0.0469 | Regression_energy: 0.4276 | Regression_eta: 0.0760 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0517 | Group Train Loss Unweighted (window mean): physical_pu: 3.7654 | physical_nopu: 4.6500 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4645 | Classification_binary: 3.5127 | Regression_cos_phi: 1.5425 | Regression_energy: 2.3632 | Regression_eta: 0.7311 | Regression_pt: 2.7511 | Regression_sin_phi: 2.3608 | LR: 3.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 4.1640s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0984 | max_forward: 0.1749 | max_loss: 0.0039 | max_backward: 0.0976 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 2.1733 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5435.1 | uss_MB: 4384.3 | child_rss_MB: 10495.1 | allocated_MB: 607.6 | reserved_MB: 28978.0 | inactive_split_MB: 330.4 +rank0 [2026-10-27 17:14:56,806] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +2 6 0x7408, 15961 49.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 51.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:18:34,347] INFO : Step 17000/100000 rank0 | Train Loss: 14.2557 | Train Loss Unweighted: 4.6663 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6292 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4540 | Regression_eta: 0.0715 | Regression_pt: 0.3998 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.7576 | physical_nopu: 4.6430 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4693 | Classification_binary: 3.6927 | Regression_cos_phi: 1.9309 | Regression_energy: 1.5646 | Regression_eta: 0.8372 | Regression_pt: 1.8225 | Regression_sin_phi: 2.0559 | LR: 3.75e-04 | DataLoad Time: 0.0033s | Model Forward Time: 4.6716s | Phase Times: max_pre_step_barrier: 0.5705 | max_batch_to_device: 0.0043 | max_forward: 0.0887 | max_loss: 0.0037 | max_backward: 0.1154 | max_optimizer_step: 0.2542 | max_tensorboard_logging: 2.1801 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5421.0 | uss_MB: 4384.2 | child_rss_MB: 10406.9 | allocated_MB: 609.7 | reserved_MB: 29500.0 | inactive_split_MB: 302.1 +rank0 [2026-10-27 17:18:34,628] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 48.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:22:19,033] INFO : Step 18000/100000 rank0 | Train Loss: 13.2896 | Train Loss Unweighted: 4.5062 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6504 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3607 | Regression_eta: 0.0732 | Regression_pt: 0.3062 | Regression_sin_phi: 0.0530 | Group Train Loss Unweighted (window mean): physical_pu: 3.7500 | physical_nopu: 4.6286 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4281 | Classification_binary: 3.6079 | Regression_cos_phi: 1.8720 | Regression_energy: 2.3319 | Regression_eta: 1.1006 | Regression_pt: 2.7773 | Regression_sin_phi: 1.5366 | LR: 3.72e-04 | DataLoad Time: 0.0016s | Model Forward Time: 10.7614s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0832 | max_loss: 0.0037 | max_backward: 0.0999 | max_optimizer_step: 0.1725 | max_tensorboard_logging: 8.8254 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 5416.7 | uss_MB: 4384.0 | child_rss_MB: 10363.7 | allocated_MB: 608.8 | reserved_MB: 29500.0 | inactive_split_MB: 196.1 +rank0 [2026-10-27 17:22:19,292] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 48.0°C 119.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 47.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:25:52,063] INFO : Step 19000/100000 rank0 | Train Loss: 15.2449 | Train Loss Unweighted: 4.8480 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5669 | Regression_cos_phi: 0.0399 | Regression_energy: 0.5792 | Regression_eta: 0.0602 | Regression_pt: 0.5415 | Regression_sin_phi: 0.0443 | Group Train Loss Unweighted (window mean): physical_pu: 3.7399 | physical_nopu: 4.6217 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5507 | Classification_binary: 3.6509 | Regression_cos_phi: 1.5233 | Regression_energy: 2.4165 | Regression_eta: 0.9729 | Regression_pt: 2.8105 | Regression_sin_phi: 1.8188 | LR: 3.69e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.8183s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0008 | max_forward: 0.0818 | max_loss: 0.0762 | max_backward: 0.1759 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 1.9822 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5410.7 | uss_MB: 4384.1 | child_rss_MB: 10330.8 | allocated_MB: 607.1 | reserved_MB: 29500.0 | inactive_split_MB: 236.2 +rank0 [2026-10-27 17:25:52,318] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +2 6 0x7408, 15961 49.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +4 8 0x7408, 13395 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 52.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:29:45,707] INFO : Step 20000/100000 rank0 | Train Loss: 13.7275 | Train Loss Unweighted: 4.6495 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.7120 | Regression_cos_phi: 0.0510 | Regression_energy: 0.4061 | Regression_eta: 0.0646 | Regression_pt: 0.3531 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.7387 | physical_nopu: 4.6038 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3326 | Classification_binary: 3.7227 | Regression_cos_phi: 1.8577 | Regression_energy: 1.7003 | Regression_eta: 1.0101 | Regression_pt: 1.8677 | Regression_sin_phi: 2.0032 | LR: 3.66e-04 | DataLoad Time: 0.0023s | Model Forward Time: 6.1722s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0007 | max_forward: 0.0851 | max_loss: 0.0039 | max_backward: 0.2016 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 4.2823 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5411.1 | uss_MB: 4383.8 | child_rss_MB: 10307.1 | allocated_MB: 609.5 | reserved_MB: 29500.0 | inactive_split_MB: 252.8 +rank0 [2026-10-27 17:29:45,967] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 46% 7% +2 6 0x7408, 15961 48.0°C 122.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +4 8 0x7408, 13395 47.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 51.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:29:46,029] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/checkpoints/checkpoint-20000.pth +rank0 [2026-10-27 17:29:46,662] INFO : Running validation on rank0 +rank0 [2026-10-27 17:29:46,662] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 17:29:46,662] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 17:29:46,662] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 17:29:46,662] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 17:29:50,464] INFO : saved batch0_epoch20000.parquet +rank0 [2026-10-27 17:29:50,645] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 17:29:50,807] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 17:29:50,893] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 17:29:50,973] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 17:29:51,052] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 17:29:51,134] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 17:29:51,214] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 17:29:51,354] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 17:29:51,491] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 17:29:51,624] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 17:29:51,704] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 17:29:51,784] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 17:29:51,865] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 17:29:51,946] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 17:29:52,027] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 17:29:52,146] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 17:29:52,281] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 17:29:52,416] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 17:29:52,495] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 17:29:52,575] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 17:29:52,655] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 17:29:52,736] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 17:29:52,817] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 17:29:52,931] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 17:29:53,042] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 17:29:53,152] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 17:29:53,229] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 17:29:53,307] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 17:29:53,384] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 17:29:53,461] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 17:29:53,542] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 17:29:53,664] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 17:29:53,799] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 17:29:53,936] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 17:29:54,018] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 17:29:54,099] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 17:29:54,178] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 17:29:54,259] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 17:29:54,340] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 17:29:54,457] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 17:29:54,595] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 17:29:54,730] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 17:29:54,809] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 17:29:54,890] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 17:29:54,967] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 17:29:55,046] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 17:29:55,129] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 17:29:55,935] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 17:29:56,046] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 17:29:56,156] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 17:29:56,231] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 17:29:56,308] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 17:29:56,385] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 17:29:56,462] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 17:29:56,539] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 17:29:56,668] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 17:29:56,799] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 17:29:56,914] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 17:29:56,989] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 17:29:57,067] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 17:29:57,143] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 17:29:57,220] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 17:29:57,297] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 17:29:57,408] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 17:29:57,486] INFO : plotted pt_target histogram +rank0 [2026-10-27 17:29:57,489] INFO : plotted pt_pred histogram +rank0 [2026-10-27 17:29:57,492] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 17:29:57,495] INFO : plotted eta_target histogram +rank0 [2026-10-27 17:29:57,498] INFO : plotted eta_pred histogram +rank0 [2026-10-27 17:29:57,501] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 17:29:57,503] INFO : plotted sphi_target histogram +rank0 [2026-10-27 17:29:57,506] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 17:29:57,509] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 17:29:57,511] INFO : plotted cphi_target histogram +rank0 [2026-10-27 17:29:57,513] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 17:29:57,516] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 17:29:57,519] INFO : plotted energy_target histogram +rank0 [2026-10-27 17:29:57,520] INFO : plotted energy_pred histogram +rank0 [2026-10-27 17:29:57,523] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 17:31:43,022] INFO : VALIDATION | Step=20000/100000 | Train Loss=13.7275 | Valid Loss=13.0842 | Train Loss Unweighted=4.6495 | Valid Loss Unweighted=4.1675 | Group Valid Loss Unweighted: physical_pu: 3.7245 | physical_nopu: 4.6105 | Stale=0 | ETA=321.8m +rank0 [2026-10-27 17:31:43,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 17:31:43,453] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 17:31:43,458] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 17:32:09,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 17:32:10,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 17:32:10,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 17:32:11,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 17:32:11,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 17:32:12,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 17:32:13,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 17:32:13,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 17:32:23,630] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 17:32:23,635] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 17:32:24,023] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 17:32:24,027] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 17:32:38,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 17:32:39,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 17:32:40,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 17:32:40,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 17:32:41,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 17:32:41,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 17:32:42,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 17:32:42,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 17:32:52,816] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 17:32:53,432] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 17:32:53,821] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 17:32:53,825] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 17:33:09,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 17:33:09,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 17:33:10,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 17:33:10,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 17:33:11,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 17:33:11,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 17:33:12,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 17:33:13,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 17:33:23,016] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 17:33:23,029] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar +rank0 [2026-10-27 17:33:28,066] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar +rank0 [2026-10-27 17:33:29,336] INFO : Plotted number of elements +rank0 [2026-10-27 17:33:31,158] INFO : Plotted jets +rank0 [2026-10-27 17:33:32,373] INFO : Plotted jet ratio +rank0 [2026-10-27 17:33:32,422] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd +rank0 [2026-10-27 17:33:37,668] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd +rank0 [2026-10-27 17:33:38,911] INFO : Plotted number of elements +rank0 [2026-10-27 17:33:41,056] INFO : Plotted jets +rank0 [2026-10-27 17:33:42,164] INFO : Plotted jet ratio +rank0 [2026-10-27 17:33:42,202] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt +rank0 [2026-10-27 17:33:47,150] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt +rank0 [2026-10-27 17:33:48,352] INFO : Plotted number of elements +rank0 [2026-10-27 17:33:50,089] INFO : Plotted jets +rank0 [2026-10-27 17:33:51,274] INFO : Plotted jet ratio +rank0 [2026-10-27 17:37:34,083] INFO : Step 21000/100000 rank0 | Train Loss: 13.1647 | Train Loss Unweighted: 4.3967 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.5106 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3742 | Regression_eta: 0.0796 | Regression_pt: 0.3229 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.7328 | physical_nopu: 4.5933 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5988 | Classification_binary: 3.4189 | Regression_cos_phi: 1.5112 | Regression_energy: 1.7114 | Regression_eta: 1.0379 | Regression_pt: 2.0047 | Regression_sin_phi: 1.9951 | LR: 3.62e-04 | DataLoad Time: 0.0022s | Model Forward Time: 4.3298s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0483 | max_forward: 0.1678 | max_loss: 0.0037 | max_backward: 0.1430 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 2.3569 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5516.5 | uss_MB: 4216.2 | child_rss_MB: 10241.6 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 238.6 +rank0 [2026-10-27 17:37:34,339] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 48.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 89% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 51.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:41:10,696] INFO : Step 22000/100000 rank0 | Train Loss: 15.3802 | Train Loss Unweighted: 4.7914 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.5311 | Regression_cos_phi: 0.0475 | Regression_energy: 0.5593 | Regression_eta: 0.0730 | Regression_pt: 0.5219 | Regression_sin_phi: 0.0434 | Group Train Loss Unweighted (window mean): physical_pu: 3.7290 | physical_nopu: 4.5858 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5114 | Classification_binary: 3.4087 | Regression_cos_phi: 1.9452 | Regression_energy: 3.2970 | Regression_eta: 1.0690 | Regression_pt: 4.0921 | Regression_sin_phi: 1.7988 | LR: 3.59e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.8474s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0905 | max_loss: 0.0038 | max_backward: 0.1361 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 2.0169 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5517.5 | uss_MB: 4498.8 | child_rss_MB: 10247.2 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 321.0 +rank0 [2026-10-27 17:41:10,954] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 97% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 51.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:44:48,364] INFO : Step 23000/100000 rank0 | Train Loss: 12.9848 | Train Loss Unweighted: 4.5226 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.7377 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3326 | Regression_eta: 0.0739 | Regression_pt: 0.2617 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.7288 | physical_nopu: 4.5774 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.1938 | Classification_binary: 3.6253 | Regression_cos_phi: 1.5732 | Regression_energy: 1.6611 | Regression_eta: 0.7284 | Regression_pt: 1.9377 | Regression_sin_phi: 2.0083 | LR: 3.55e-04 | DataLoad Time: 0.0068s | Model Forward Time: 3.8685s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0994 | max_loss: 0.0039 | max_backward: 0.1244 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.9838 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 5517.5 | uss_MB: 4498.0 | child_rss_MB: 10251.2 | allocated_MB: 608.2 | reserved_MB: 29422.0 | inactive_split_MB: 218.5 +rank0 [2026-10-27 17:44:48,659] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +6 10 0x7408, 62036 52.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:48:44,725] INFO : Step 24000/100000 rank0 | Train Loss: 16.3896 | Train Loss Unweighted: 4.8949 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4510 | Regression_cos_phi: 0.0465 | Regression_energy: 0.6613 | Regression_eta: 0.0617 | Regression_pt: 0.6132 | Regression_sin_phi: 0.0453 | Group Train Loss Unweighted (window mean): physical_pu: 3.7170 | physical_nopu: 4.5767 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3666 | Classification_binary: 3.5135 | Regression_cos_phi: 2.4820 | Regression_energy: 3.7039 | Regression_eta: 1.0675 | Regression_pt: 4.5944 | Regression_sin_phi: 1.8374 | LR: 3.51e-04 | DataLoad Time: 0.0027s | Model Forward Time: 3.8026s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0398 | max_forward: 0.1314 | max_loss: 0.0037 | max_backward: 0.0994 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 1.8290 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5518.7 | uss_MB: 4498.4 | child_rss_MB: 10238.8 | allocated_MB: 606.3 | reserved_MB: 29422.0 | inactive_split_MB: 288.6 +rank0 [2026-10-27 17:48:44,987] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 50.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +6 10 0x7408, 62036 52.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:52:29,788] INFO : Step 25000/100000 rank0 | Train Loss: 13.0062 | Train Loss Unweighted: 4.4048 | Train Loss Components: Classification: 0.0114 | Classification_binary: 3.6384 | Regression_cos_phi: 0.0552 | Regression_energy: 0.3153 | Regression_eta: 0.0680 | Regression_pt: 0.2574 | Regression_sin_phi: 0.0590 | Group Train Loss Unweighted (window mean): physical_pu: 3.7138 | physical_nopu: 4.5644 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3895 | Classification_binary: 3.7513 | Regression_cos_phi: 2.3274 | Regression_energy: 1.4592 | Regression_eta: 1.1573 | Regression_pt: 1.4981 | Regression_sin_phi: 2.3060 | LR: 3.47e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.2863s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0007 | max_forward: 0.1394 | max_loss: 0.0036 | max_backward: 0.2004 | max_optimizer_step: 0.1603 | max_tensorboard_logging: 1.3791 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5517.7 | uss_MB: 4498.3 | child_rss_MB: 10255.6 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 213.9 +rank0 [2026-10-27 17:52:30,044] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 50.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 49.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +6 10 0x7408, 62036 53.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:56:11,317] INFO : Step 26000/100000 rank0 | Train Loss: 13.5469 | Train Loss Unweighted: 4.4523 | Train Loss Components: Classification: 0.0185 | Classification_binary: 3.5246 | Regression_cos_phi: 0.0498 | Regression_energy: 0.3935 | Regression_eta: 0.0732 | Regression_pt: 0.3461 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5624 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6131 | Classification_binary: 3.5950 | Regression_cos_phi: 1.9830 | Regression_energy: 1.7663 | Regression_eta: 1.3261 | Regression_pt: 1.9931 | Regression_sin_phi: 2.0192 | LR: 3.43e-04 | DataLoad Time: 0.0012s | Model Forward Time: 3.8045s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0838 | max_forward: 0.1679 | max_loss: 0.0039 | max_backward: 0.0991 | max_optimizer_step: 0.2674 | max_tensorboard_logging: 1.6325 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5515.7 | uss_MB: 4498.4 | child_rss_MB: 10248.9 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 201.9 +rank0 [2026-10-27 17:56:11,573] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 1% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 70% +6 10 0x7408, 62036 52.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 17:59:49,272] INFO : Step 27000/100000 rank0 | Train Loss: 17.1780 | Train Loss Unweighted: 5.0531 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.5146 | Regression_cos_phi: 0.0448 | Regression_energy: 0.6955 | Regression_eta: 0.0778 | Regression_pt: 0.6583 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5579 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4172 | Classification_binary: 3.5846 | Regression_cos_phi: 1.9660 | Regression_energy: 2.6473 | Regression_eta: 1.6236 | Regression_pt: 3.2232 | Regression_sin_phi: 1.8858 | LR: 3.39e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.0895s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1019 | max_loss: 0.0489 | max_backward: 0.1596 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5507.7 | uss_MB: 4499.5 | child_rss_MB: 10244.8 | allocated_MB: 611.7 | reserved_MB: 30248.0 | inactive_split_MB: 336.3 +rank0 [2026-10-27 17:59:49,529] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 48.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +6 10 0x7408, 62036 52.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:03:27,391] INFO : Step 28000/100000 rank0 | Train Loss: 12.6721 | Train Loss Unweighted: 4.3436 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5550 | Regression_cos_phi: 0.0488 | Regression_energy: 0.3264 | Regression_eta: 0.0754 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7057 | physical_nopu: 4.5482 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4846 | Classification_binary: 3.6752 | Regression_cos_phi: 1.9194 | Regression_energy: 1.8741 | Regression_eta: 1.3226 | Regression_pt: 2.2752 | Regression_sin_phi: 2.5357 | LR: 3.35e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.6492s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0898 | max_loss: 0.0057 | max_backward: 0.1164 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.8021 | max_tensorboard_flush: 0.0040 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 5489.8 | uss_MB: 4496.9 | child_rss_MB: 10097.2 | allocated_MB: 606.6 | reserved_MB: 30248.0 | inactive_split_MB: 278.7 +rank0 [2026-10-27 18:03:27,674] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +2 6 0x7408, 15961 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +6 10 0x7408, 62036 52.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:07:26,044] INFO : Step 29000/100000 rank0 | Train Loss: 14.8403 | Train Loss Unweighted: 4.5753 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.4458 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4901 | Regression_eta: 0.0767 | Regression_pt: 0.4459 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.7068 | physical_nopu: 4.5439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5741 | Classification_binary: 3.4733 | Regression_cos_phi: 1.8550 | Regression_energy: 2.1228 | Regression_eta: 1.2183 | Regression_pt: 2.5958 | Regression_sin_phi: 1.9581 | LR: 3.30e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.9543s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1464 | max_loss: 0.0037 | max_backward: 0.0935 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 1.1535 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5489.8 | uss_MB: 4498.4 | child_rss_MB: 10094.9 | allocated_MB: 607.4 | reserved_MB: 30248.0 | inactive_split_MB: 313.3 +rank0 [2026-10-27 18:07:26,303] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 49.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 48.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +6 10 0x7408, 62036 52.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:11:13,176] INFO : Step 30000/100000 rank0 | Train Loss: 13.2916 | Train Loss Unweighted: 4.4954 | Train Loss Components: Classification: 0.0126 | Classification_binary: 3.5989 | Regression_cos_phi: 0.0482 | Regression_energy: 0.3865 | Regression_eta: 0.0707 | Regression_pt: 0.3291 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7005 | physical_nopu: 4.5378 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3783 | Classification_binary: 3.5178 | Regression_cos_phi: 1.7354 | Regression_energy: 1.8196 | Regression_eta: 1.2258 | Regression_pt: 1.9494 | Regression_sin_phi: 1.5874 | LR: 3.26e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.3806s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0805 | max_loss: 0.0038 | max_backward: 0.0926 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 1.5279 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 5479.8 | uss_MB: 4498.3 | child_rss_MB: 10016.6 | allocated_MB: 608.2 | reserved_MB: 30760.0 | inactive_split_MB: 232.9 +rank0 [2026-10-27 18:11:13,429] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 132.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 48.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +6 10 0x7408, 62036 51.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:11:13,468] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/checkpoints/checkpoint-30000.pth +rank0 [2026-10-27 18:11:14,049] INFO : Running validation on rank0 +rank0 [2026-10-27 18:11:14,050] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 18:11:14,050] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 18:11:14,050] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 18:11:14,050] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 18:11:17,738] INFO : saved batch0_epoch30000.parquet +rank0 [2026-10-27 18:11:17,908] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 18:11:18,066] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 18:11:18,148] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 18:11:18,228] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 18:11:18,309] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 18:11:18,389] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 18:11:18,469] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 18:11:18,607] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 18:11:18,741] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 18:11:18,875] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 18:11:18,956] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 18:11:19,037] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 18:11:19,116] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 18:11:19,197] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 18:11:19,278] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 18:11:19,400] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 18:11:19,525] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 18:11:19,650] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 18:11:19,732] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 18:11:19,814] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 18:11:19,892] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 18:11:19,972] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 18:11:20,053] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 18:11:20,171] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 18:11:20,282] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 18:11:20,391] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 18:11:20,471] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 18:11:20,548] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 18:11:20,624] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 18:11:20,702] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 18:11:20,777] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 18:11:20,903] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 18:11:21,035] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 18:11:21,168] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 18:11:21,247] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 18:11:21,328] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 18:11:21,405] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 18:11:21,483] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 18:11:21,563] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 18:11:21,682] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 18:11:21,818] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 18:11:21,957] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 18:11:22,038] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 18:11:22,119] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 18:11:22,196] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 18:11:22,913] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 18:11:22,993] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 18:11:23,128] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 18:11:23,246] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 18:11:23,362] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 18:11:23,439] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 18:11:23,518] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 18:11:23,599] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 18:11:23,679] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 18:11:23,760] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 18:11:23,894] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 18:11:24,012] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 18:11:24,129] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 18:11:24,206] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 18:11:24,284] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 18:11:24,363] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 18:11:24,444] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 18:11:24,522] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 18:11:24,658] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 18:11:24,665] INFO : plotted pt_target histogram +rank0 [2026-10-27 18:11:24,668] INFO : plotted pt_pred histogram +rank0 [2026-10-27 18:11:24,671] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 18:11:24,674] INFO : plotted eta_target histogram +rank0 [2026-10-27 18:11:24,676] INFO : plotted eta_pred histogram +rank0 [2026-10-27 18:11:24,679] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 18:11:24,680] INFO : plotted sphi_target histogram +rank0 [2026-10-27 18:11:24,683] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 18:11:24,686] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 18:11:24,689] INFO : plotted cphi_target histogram +rank0 [2026-10-27 18:11:24,691] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 18:11:24,694] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 18:11:24,697] INFO : plotted energy_target histogram +rank0 [2026-10-27 18:11:24,700] INFO : plotted energy_pred histogram +rank0 [2026-10-27 18:11:24,703] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 18:13:08,025] INFO : VALIDATION | Step=30000/100000 | Train Loss=13.2916 | Valid Loss=12.9314 | Train Loss Unweighted=4.4954 | Valid Loss Unweighted=4.1143 | Group Valid Loss Unweighted: physical_pu: 3.6878 | physical_nopu: 4.5409 | Stale=0 | ETA=284.3m +rank0 [2026-10-27 18:13:08,034] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 18:13:08,190] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 18:13:08,193] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 18:13:34,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 18:13:35,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 18:13:35,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 18:13:36,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 18:13:36,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 18:13:37,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 18:13:37,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 18:13:38,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 18:13:48,476] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 18:13:48,508] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 18:13:49,355] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 18:13:49,359] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 18:14:04,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 18:14:05,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 18:14:05,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 18:14:06,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 18:14:06,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 18:14:07,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 18:14:07,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 18:14:08,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 18:14:18,295] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-27 18:14:18,746] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 18:14:19,218] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 18:14:19,222] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 18:14:34,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 18:14:34,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 18:14:35,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 18:14:35,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 18:14:36,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 18:14:37,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 18:14:37,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 18:14:38,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 18:14:48,104] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 18:14:48,138] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar +rank0 [2026-10-27 18:14:53,163] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar +rank0 [2026-10-27 18:14:54,314] INFO : Plotted number of elements +rank0 [2026-10-27 18:14:56,024] INFO : Plotted jets +rank0 [2026-10-27 18:14:57,216] INFO : Plotted jet ratio +rank0 [2026-10-27 18:14:57,261] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd +rank0 [2026-10-27 18:15:02,472] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd +rank0 [2026-10-27 18:15:04,145] INFO : Plotted number of elements +rank0 [2026-10-27 18:15:05,817] INFO : Plotted jets +rank0 [2026-10-27 18:15:06,922] INFO : Plotted jet ratio +rank0 [2026-10-27 18:15:06,960] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt +rank0 [2026-10-27 18:15:11,858] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt +rank0 [2026-10-27 18:15:13,051] INFO : Plotted number of elements +rank0 [2026-10-27 18:15:14,767] INFO : Plotted jets +rank0 [2026-10-27 18:15:15,993] INFO : Plotted jet ratio +rank0 [2026-10-27 18:19:00,999] INFO : Step 31000/100000 rank0 | Train Loss: 12.7750 | Train Loss Unweighted: 4.3889 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5406 | Regression_cos_phi: 0.0465 | Regression_energy: 0.3622 | Regression_eta: 0.0700 | Regression_pt: 0.3101 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6964 | physical_nopu: 4.5360 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4321 | Classification_binary: 3.5663 | Regression_cos_phi: 1.7193 | Regression_energy: 1.5889 | Regression_eta: 0.8746 | Regression_pt: 1.7293 | Regression_sin_phi: 1.5898 | LR: 3.21e-04 | DataLoad Time: 0.0019s | Model Forward Time: 4.0173s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.2346 | max_loss: 0.0037 | max_backward: 0.1255 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 1.8384 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1395 | Memory: rss_MB: 6773.5 | uss_MB: 5791.8 | child_rss_MB: 10011.8 | allocated_MB: 609.3 | reserved_MB: 29142.0 | inactive_split_MB: 278.5 +rank0 [2026-10-27 18:19:01,283] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:22:45,492] INFO : Step 32000/100000 rank0 | Train Loss: 14.3617 | Train Loss Unweighted: 4.6225 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4934 | Regression_cos_phi: 0.0448 | Regression_energy: 0.5043 | Regression_eta: 0.0718 | Regression_pt: 0.4507 | Regression_sin_phi: 0.0414 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5314 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4557 | Classification_binary: 3.4409 | Regression_cos_phi: 1.3281 | Regression_energy: 2.6131 | Regression_eta: 1.0537 | Regression_pt: 3.2409 | Regression_sin_phi: 1.5644 | LR: 3.16e-04 | DataLoad Time: 0.0104s | Model Forward Time: 3.1944s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1373 | max_loss: 0.0041 | max_backward: 0.2040 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 1.2065 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5556.1 | uss_MB: 4574.8 | child_rss_MB: 10008.9 | allocated_MB: 606.1 | reserved_MB: 29142.0 | inactive_split_MB: 511.2 +rank0 [2026-10-27 18:22:45,749] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 3% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 50.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +4 8 0x7408, 13395 49.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 53.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:26:33,631] INFO : Step 33000/100000 rank0 | Train Loss: 12.0000 | Train Loss Unweighted: 3.7115 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9674 | Regression_cos_phi: 0.0474 | Regression_energy: 0.2863 | Regression_eta: 0.1056 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4604 | Classification_binary: 3.0230 | Regression_cos_phi: 1.8286 | Regression_energy: 1.4491 | Regression_eta: 1.6652 | Regression_pt: 1.6188 | Regression_sin_phi: 1.9359 | LR: 3.12e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.2001s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0030 | max_forward: 0.1619 | max_loss: 0.0119 | max_backward: 0.2845 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.1927 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5557.1 | uss_MB: 4556.3 | child_rss_MB: 10013.2 | allocated_MB: 623.7 | reserved_MB: 29572.0 | inactive_split_MB: 1441.5 +rank0 [2026-10-27 18:26:33,887] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +4 8 0x7408, 13395 48.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 53.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:30:16,608] INFO : Step 34000/100000 rank0 | Train Loss: 15.0627 | Train Loss Unweighted: 4.6899 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4670 | Regression_cos_phi: 0.0454 | Regression_energy: 0.5438 | Regression_eta: 0.0626 | Regression_pt: 0.5082 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6889 | physical_nopu: 4.5215 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3751 | Classification_binary: 3.6084 | Regression_cos_phi: 1.4220 | Regression_energy: 2.7544 | Regression_eta: 0.9070 | Regression_pt: 3.3496 | Regression_sin_phi: 1.5969 | LR: 3.07e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.0097s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0007 | max_forward: 0.1346 | max_loss: 0.0037 | max_backward: 0.1938 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1506 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5552.6 | uss_MB: 4573.6 | child_rss_MB: 9984.9 | allocated_MB: 609.6 | reserved_MB: 29572.0 | inactive_split_MB: 230.4 +rank0 [2026-10-27 18:30:16,869] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 49.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 48.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:33:55,887] INFO : Step 35000/100000 rank0 | Train Loss: 12.2019 | Train Loss Unweighted: 4.3540 | Train Loss Components: Classification: 0.0089 | Classification_binary: 3.6426 | Regression_cos_phi: 0.0532 | Regression_energy: 0.2965 | Regression_eta: 0.0664 | Regression_pt: 0.2370 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6854 | physical_nopu: 4.5184 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2470 | Classification_binary: 3.8784 | Regression_cos_phi: 1.9874 | Regression_energy: 1.6235 | Regression_eta: 0.9444 | Regression_pt: 1.7274 | Regression_sin_phi: 1.8245 | LR: 3.02e-04 | DataLoad Time: 0.0500s | Model Forward Time: 3.2111s | Phase Times: max_pre_step_barrier: 0.0511 | max_batch_to_device: 0.0857 | max_forward: 0.1832 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 1.2656 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 5548.6 | uss_MB: 4574.4 | child_rss_MB: 9918.7 | allocated_MB: 608.2 | reserved_MB: 29572.0 | inactive_split_MB: 245.8 +rank0 [2026-10-27 18:33:56,142] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 49.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 48.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:37:34,533] INFO : Step 36000/100000 rank0 | Train Loss: 12.7130 | Train Loss Unweighted: 4.3877 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.5479 | Regression_cos_phi: 0.0433 | Regression_energy: 0.3521 | Regression_eta: 0.0636 | Regression_pt: 0.3167 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.6860 | physical_nopu: 4.5137 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6365 | Classification_binary: 3.5916 | Regression_cos_phi: 1.5402 | Regression_energy: 2.2597 | Regression_eta: 1.0619 | Regression_pt: 2.6068 | Regression_sin_phi: 2.1562 | LR: 2.97e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.2617s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0007 | max_forward: 0.1685 | max_loss: 0.0137 | max_backward: 0.0851 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.2906 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 5532.6 | uss_MB: 4487.8 | child_rss_MB: 9818.0 | allocated_MB: 609.0 | reserved_MB: 29572.0 | inactive_split_MB: 268.4 +rank0 [2026-10-27 18:37:34,803] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 48.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 52.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 40% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:41:13,188] INFO : Step 37000/100000 rank0 | Train Loss: 16.1273 | Train Loss Unweighted: 4.7522 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4049 | Regression_cos_phi: 0.0495 | Regression_energy: 0.5975 | Regression_eta: 0.0881 | Regression_pt: 0.5494 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6829 | physical_nopu: 4.5107 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4634 | Classification_binary: 3.5822 | Regression_cos_phi: 2.4891 | Regression_energy: 2.0886 | Regression_eta: 1.4711 | Regression_pt: 2.2060 | Regression_sin_phi: 2.3121 | LR: 2.91e-04 | DataLoad Time: 0.0029s | Model Forward Time: 2.9160s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0884 | max_loss: 0.0036 | max_backward: 0.0963 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 1.0635 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0656 | Memory: rss_MB: 5529.2 | uss_MB: 4574.6 | child_rss_MB: 9790.1 | allocated_MB: 605.6 | reserved_MB: 29572.0 | inactive_split_MB: 233.2 +rank0 [2026-10-27 18:41:13,452] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 49.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +4 8 0x7408, 13395 48.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 52.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:44:44,971] INFO : Step 38000/100000 rank0 | Train Loss: 12.5142 | Train Loss Unweighted: 4.4218 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.6290 | Regression_cos_phi: 0.0475 | Regression_energy: 0.3376 | Regression_eta: 0.0660 | Regression_pt: 0.2827 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6875 | physical_nopu: 4.5136 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2896 | Classification_binary: 3.5323 | Regression_cos_phi: 1.8563 | Regression_energy: 1.7174 | Regression_eta: 1.3646 | Regression_pt: 2.1494 | Regression_sin_phi: 2.0190 | LR: 2.86e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.1180s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.0807 | max_loss: 0.0039 | max_backward: 0.1583 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.2386 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5519.2 | uss_MB: 4574.3 | child_rss_MB: 9713.9 | allocated_MB: 607.7 | reserved_MB: 29572.0 | inactive_split_MB: 203.1 +rank0 [2026-10-27 18:44:45,231] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 49.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 48.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:48:30,874] INFO : Step 39000/100000 rank0 | Train Loss: 13.9834 | Train Loss Unweighted: 4.4509 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4391 | Regression_cos_phi: 0.0477 | Regression_energy: 0.4363 | Regression_eta: 0.0792 | Regression_pt: 0.3839 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.6802 | physical_nopu: 4.5058 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5306 | Classification_binary: 3.5240 | Regression_cos_phi: 2.3367 | Regression_energy: 2.7250 | Regression_eta: 1.2272 | Regression_pt: 2.9517 | Regression_sin_phi: 2.4097 | LR: 2.81e-04 | DataLoad Time: 0.0026s | Model Forward Time: 4.5408s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.0950 | max_loss: 0.0039 | max_backward: 0.1102 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 2.6970 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5506.6 | uss_MB: 4574.6 | child_rss_MB: 9623.2 | allocated_MB: 611.4 | reserved_MB: 29682.0 | inactive_split_MB: 276.6 +rank0 [2026-10-27 18:48:31,138] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 3% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +2 6 0x7408, 15961 48.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +4 8 0x7408, 13395 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +6 10 0x7408, 62036 51.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:52:08,423] INFO : Step 40000/100000 rank0 | Train Loss: 15.4467 | Train Loss Unweighted: 4.8241 | Train Loss Components: Classification: 0.0121 | Classification_binary: 3.5538 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5661 | Regression_eta: 0.0744 | Regression_pt: 0.5256 | Regression_sin_phi: 0.0462 | Group Train Loss Unweighted (window mean): physical_pu: 3.6754 | physical_nopu: 4.5045 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3218 | Classification_binary: 3.6854 | Regression_cos_phi: 1.6847 | Regression_energy: 2.0447 | Regression_eta: 1.1713 | Regression_pt: 2.2282 | Regression_sin_phi: 2.0674 | LR: 2.76e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.0698s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.1251 | max_loss: 0.0037 | max_backward: 0.0913 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2799 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5491.6 | uss_MB: 4574.5 | child_rss_MB: 9510.6 | allocated_MB: 607.6 | reserved_MB: 29682.0 | inactive_split_MB: 251.4 +rank0 [2026-10-27 18:52:08,681] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +2 6 0x7408, 15961 49.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 98% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +4 8 0x7408, 13395 49.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 52.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 18:52:08,800] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/checkpoints/checkpoint-40000.pth +rank0 [2026-10-27 18:52:09,355] INFO : Running validation on rank0 +rank0 [2026-10-27 18:52:09,356] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 18:52:09,356] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 18:52:09,356] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 18:52:09,356] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 18:52:13,094] INFO : saved batch0_epoch40000.parquet +rank0 [2026-10-27 18:52:13,272] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 18:52:13,439] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 18:52:13,524] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 18:52:13,609] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 18:52:13,691] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 18:52:13,773] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 18:52:13,853] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 18:52:13,990] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 18:52:14,127] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 18:52:14,262] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 18:52:14,341] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 18:52:14,422] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 18:52:14,501] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 18:52:14,585] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 18:52:14,668] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 18:52:14,799] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 18:52:14,932] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 18:52:15,055] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 18:52:15,137] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 18:52:15,217] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 18:52:15,298] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 18:52:15,376] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 18:52:15,453] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 18:52:15,563] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 18:52:15,691] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 18:52:15,812] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 18:52:15,890] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 18:52:15,969] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 18:52:16,045] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 18:52:16,122] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 18:52:16,204] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 18:52:16,325] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 18:52:16,468] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 18:52:16,604] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 18:52:16,685] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 18:52:16,769] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 18:52:16,851] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 18:52:16,931] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 18:52:17,013] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 18:52:17,136] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 18:52:17,278] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 18:52:17,414] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 18:52:17,495] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 18:52:17,573] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 18:52:17,651] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 18:52:17,731] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 18:52:17,810] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 18:52:17,948] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 18:52:18,767] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 18:52:18,884] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 18:52:18,961] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 18:52:19,037] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 18:52:19,118] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 18:52:19,200] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 18:52:19,281] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 18:52:19,397] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 18:52:19,517] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 18:52:19,635] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 18:52:19,714] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 18:52:19,793] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 18:52:19,874] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 18:52:19,954] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 18:52:20,034] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 18:52:20,171] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 18:52:20,178] INFO : plotted pt_target histogram +rank0 [2026-10-27 18:52:20,182] INFO : plotted pt_pred histogram +rank0 [2026-10-27 18:52:20,185] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 18:52:20,197] INFO : plotted eta_target histogram +rank0 [2026-10-27 18:52:20,200] INFO : plotted eta_pred histogram +rank0 [2026-10-27 18:52:20,203] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 18:52:20,217] INFO : plotted sphi_target histogram +rank0 [2026-10-27 18:52:20,220] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 18:52:20,225] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 18:52:20,237] INFO : plotted cphi_target histogram +rank0 [2026-10-27 18:52:20,240] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 18:52:20,243] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 18:52:20,246] INFO : plotted energy_target histogram +rank0 [2026-10-27 18:52:20,249] INFO : plotted energy_pred histogram +rank0 [2026-10-27 18:52:20,252] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 18:54:12,275] INFO : VALIDATION | Step=40000/100000 | Train Loss=15.4467 | Valid Loss=12.8401 | Train Loss Unweighted=4.8241 | Valid Loss Unweighted=4.0861 | Group Valid Loss Unweighted: physical_pu: 3.6687 | physical_nopu: 4.5035 | Stale=0 | ETA=244.4m +rank0 [2026-10-27 18:54:12,283] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 18:54:12,446] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 18:54:12,450] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 18:54:37,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 18:54:37,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 18:54:38,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 18:54:39,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 18:54:39,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 18:54:40,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 18:54:40,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 18:54:41,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 18:54:51,311] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 18:54:51,317] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 18:54:51,717] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 18:54:51,721] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 18:55:07,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 18:55:07,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 18:55:08,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 18:55:08,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 18:55:09,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 18:55:09,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 18:55:10,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 18:55:10,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 18:55:20,904] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 18:55:21,167] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 18:55:21,563] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 18:55:21,567] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 18:55:37,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 18:55:37,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 18:55:38,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 18:55:38,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 18:55:39,354] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 18:55:39,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 18:55:40,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 18:55:40,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 18:55:50,935] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 18:55:50,947] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar +rank0 [2026-10-27 18:55:55,947] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar +rank0 [2026-10-27 18:55:57,156] INFO : Plotted number of elements +rank0 [2026-10-27 18:55:58,847] INFO : Plotted jets +rank0 [2026-10-27 18:56:00,056] INFO : Plotted jet ratio +rank0 [2026-10-27 18:56:00,152] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd +rank0 [2026-10-27 18:56:05,385] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd +rank0 [2026-10-27 18:56:07,137] INFO : Plotted number of elements +rank0 [2026-10-27 18:56:08,815] INFO : Plotted jets +rank0 [2026-10-27 18:56:09,942] INFO : Plotted jet ratio +rank0 [2026-10-27 18:56:09,980] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt +rank0 [2026-10-27 18:56:15,015] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt +rank0 [2026-10-27 18:56:16,236] INFO : Plotted number of elements +rank0 [2026-10-27 18:56:17,920] INFO : Plotted jets +rank0 [2026-10-27 18:56:19,117] INFO : Plotted jet ratio +rank0 [2026-10-27 19:00:09,086] INFO : Step 41000/100000 rank0 | Train Loss: 13.2101 | Train Loss Unweighted: 4.4466 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5620 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3777 | Regression_eta: 0.0720 | Regression_pt: 0.3238 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6774 | physical_nopu: 4.4925 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4342 | Classification_binary: 3.6669 | Regression_cos_phi: 1.7096 | Regression_energy: 2.8468 | Regression_eta: 0.8304 | Regression_pt: 3.3265 | Regression_sin_phi: 2.0830 | LR: 2.70e-04 | DataLoad Time: 0.0017s | Model Forward Time: 4.1500s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0904 | max_forward: 0.2227 | max_loss: 0.0038 | max_backward: 0.1239 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 2.1580 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6748.4 | uss_MB: 5831.9 | child_rss_MB: 9514.0 | allocated_MB: 608.3 | reserved_MB: 27230.0 | inactive_split_MB: 387.2 +rank0 [2026-10-27 19:00:09,346] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:03:46,601] INFO : Step 42000/100000 rank0 | Train Loss: 13.7061 | Train Loss Unweighted: 4.4760 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4108 | Regression_cos_phi: 0.0459 | Regression_energy: 0.4702 | Regression_eta: 0.0614 | Regression_pt: 0.4341 | Regression_sin_phi: 0.0374 | Group Train Loss Unweighted (window mean): physical_pu: 3.6711 | physical_nopu: 4.4967 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5606 | Classification_binary: 3.3326 | Regression_cos_phi: 1.7354 | Regression_energy: 3.3236 | Regression_eta: 1.0699 | Regression_pt: 4.2309 | Regression_sin_phi: 1.2612 | LR: 2.65e-04 | DataLoad Time: 0.0086s | Model Forward Time: 3.7297s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0008 | max_forward: 0.1920 | max_loss: 0.0038 | max_backward: 0.1427 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 1.6213 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6739.4 | uss_MB: 5628.8 | child_rss_MB: 9436.4 | allocated_MB: 609.4 | reserved_MB: 27632.0 | inactive_split_MB: 509.5 +rank0 [2026-10-27 19:03:46,890] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 132.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 48.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:07:24,187] INFO : Step 43000/100000 rank0 | Train Loss: 11.9489 | Train Loss Unweighted: 3.6938 | Train Loss Components: Classification: 0.0162 | Classification_binary: 2.9682 | Regression_cos_phi: 0.0472 | Regression_energy: 0.2764 | Regression_eta: 0.1089 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6702 | physical_nopu: 4.4944 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5271 | Classification_binary: 3.0520 | Regression_cos_phi: 1.7560 | Regression_energy: 1.4376 | Regression_eta: 1.4678 | Regression_pt: 1.5752 | Regression_sin_phi: 1.8828 | LR: 2.59e-04 | DataLoad Time: 0.0018s | Model Forward Time: 4.5104s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0032 | max_forward: 0.2195 | max_loss: 0.0114 | max_backward: 0.4513 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 2.0889 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0865 | Memory: rss_MB: 5560.5 | uss_MB: 3524.0 | child_rss_MB: 9348.2 | allocated_MB: 619.9 | reserved_MB: 27918.0 | inactive_split_MB: 1305.5 +rank0 [2026-10-27 19:07:24,445] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 1% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 48.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 58% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 45% 1% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 1% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 51.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:11:15,497] INFO : Step 44000/100000 rank0 | Train Loss: 15.8790 | Train Loss Unweighted: 4.7893 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4610 | Regression_cos_phi: 0.0438 | Regression_energy: 0.5944 | Regression_eta: 0.0747 | Regression_pt: 0.5488 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6681 | physical_nopu: 4.4856 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6777 | Classification_binary: 3.2922 | Regression_cos_phi: 1.9570 | Regression_energy: 5.0829 | Regression_eta: 1.5313 | Regression_pt: 6.1105 | Regression_sin_phi: 2.3213 | LR: 2.54e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.8074s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0980 | max_loss: 0.0531 | max_backward: 0.1371 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.9403 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5553.5 | uss_MB: 4666.8 | child_rss_MB: 9287.7 | allocated_MB: 606.4 | reserved_MB: 31154.0 | inactive_split_MB: 295.0 +rank0 [2026-10-27 19:11:15,764] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 50% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 48.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 51.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:15:02,041] INFO : Step 45000/100000 rank0 | Train Loss: 13.8786 | Train Loss Unweighted: 4.5241 | Train Loss Components: Classification: 0.0100 | Classification_binary: 3.5822 | Regression_cos_phi: 0.0536 | Regression_energy: 0.4068 | Regression_eta: 0.0847 | Regression_pt: 0.3344 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 3.6667 | physical_nopu: 4.4836 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2845 | Classification_binary: 3.5044 | Regression_cos_phi: 1.7902 | Regression_energy: 2.2411 | Regression_eta: 1.1004 | Regression_pt: 2.2364 | Regression_sin_phi: 2.1303 | LR: 2.48e-04 | DataLoad Time: 0.0019s | Model Forward Time: 5.2599s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0009 | max_forward: 0.0951 | max_loss: 0.0815 | max_backward: 0.1837 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 3.3466 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 5551.4 | uss_MB: 4666.0 | child_rss_MB: 9284.1 | allocated_MB: 608.8 | reserved_MB: 31154.0 | inactive_split_MB: 289.8 +rank0 [2026-10-27 19:15:02,300] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 50% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 48.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 47.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 51.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:18:43,374] INFO : Step 46000/100000 rank0 | Train Loss: 13.1587 | Train Loss Unweighted: 4.3779 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.4817 | Regression_cos_phi: 0.0459 | Regression_energy: 0.3805 | Regression_eta: 0.0668 | Regression_pt: 0.3375 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.6620 | physical_nopu: 4.4831 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4816 | Classification_binary: 3.5560 | Regression_cos_phi: 1.6501 | Regression_energy: 2.1024 | Regression_eta: 1.1881 | Regression_pt: 2.5428 | Regression_sin_phi: 1.8764 | LR: 2.43e-04 | DataLoad Time: 0.0015s | Model Forward Time: 3.9885s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0924 | max_forward: 0.1910 | max_loss: 0.0039 | max_backward: 0.1028 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 2.0151 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5545.4 | uss_MB: 4666.2 | child_rss_MB: 9248.1 | allocated_MB: 609.2 | reserved_MB: 31154.0 | inactive_split_MB: 277.4 +rank0 [2026-10-27 19:18:43,627] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 50% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:22:32,168] INFO : Step 47000/100000 rank0 | Train Loss: 15.2118 | Train Loss Unweighted: 4.6831 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3624 | Regression_cos_phi: 0.0409 | Regression_energy: 0.5987 | Regression_eta: 0.0659 | Regression_pt: 0.5598 | Regression_sin_phi: 0.0405 | Group Train Loss Unweighted (window mean): physical_pu: 3.6609 | physical_nopu: 4.4795 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4507 | Classification_binary: 3.6289 | Regression_cos_phi: 1.3238 | Regression_energy: 3.5265 | Regression_eta: 1.0374 | Regression_pt: 4.5373 | Regression_sin_phi: 1.7262 | LR: 2.37e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.9177s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0777 | max_forward: 0.1864 | max_loss: 0.0077 | max_backward: 0.1105 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 1.8392 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5533.2 | uss_MB: 4665.8 | child_rss_MB: 9129.7 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 305.0 +rank0 [2026-10-27 19:22:32,437] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 50% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 48.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:26:21,817] INFO : Step 48000/100000 rank0 | Train Loss: 12.0455 | Train Loss Unweighted: 4.3014 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.6150 | Regression_cos_phi: 0.0526 | Regression_energy: 0.2805 | Regression_eta: 0.0652 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6617 | physical_nopu: 4.4726 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2701 | Classification_binary: 3.5694 | Regression_cos_phi: 2.0982 | Regression_energy: 1.3637 | Regression_eta: 0.8520 | Regression_pt: 1.5277 | Regression_sin_phi: 1.7474 | LR: 2.31e-04 | DataLoad Time: 0.0015s | Model Forward Time: 4.1220s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0531 | max_forward: 0.1607 | max_loss: 0.0044 | max_backward: 0.0997 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 2.0345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5534.2 | uss_MB: 4664.3 | child_rss_MB: 9129.2 | allocated_MB: 606.8 | reserved_MB: 31154.0 | inactive_split_MB: 262.3 +rank0 [2026-10-27 19:26:22,073] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 132.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 50% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 49.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 23% +6 10 0x7408, 62036 51.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:30:06,953] INFO : Step 49000/100000 rank0 | Train Loss: 13.3973 | Train Loss Unweighted: 4.4035 | Train Loss Components: Classification: 0.0168 | Classification_binary: 3.5033 | Regression_cos_phi: 0.0506 | Regression_energy: 0.3797 | Regression_eta: 0.0752 | Regression_pt: 0.3299 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6566 | physical_nopu: 4.4735 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4493 | Classification_binary: 3.7147 | Regression_cos_phi: 1.8120 | Regression_energy: 1.7680 | Regression_eta: 1.3141 | Regression_pt: 2.0793 | Regression_sin_phi: 2.0286 | LR: 2.26e-04 | DataLoad Time: 0.0020s | Model Forward Time: 5.2857s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0220 | max_forward: 0.1075 | max_loss: 0.0037 | max_backward: 0.1020 | max_optimizer_step: 0.2722 | max_tensorboard_logging: 3.2606 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5532.9 | uss_MB: 4389.0 | child_rss_MB: 9133.5 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 312.3 +rank0 [2026-10-27 19:30:07,213] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 50% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 49.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 48.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 51.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:33:48,468] INFO : Step 50000/100000 rank0 | Train Loss: 14.2027 | Train Loss Unweighted: 4.5299 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4272 | Regression_cos_phi: 0.0432 | Regression_energy: 0.4877 | Regression_eta: 0.0670 | Regression_pt: 0.4429 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4710 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5174 | Classification_binary: 3.2955 | Regression_cos_phi: 1.7817 | Regression_energy: 2.4772 | Regression_eta: 1.0918 | Regression_pt: 2.9492 | Regression_sin_phi: 1.6222 | LR: 2.20e-04 | DataLoad Time: 0.0015s | Model Forward Time: 3.6905s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0964 | max_loss: 0.0590 | max_backward: 0.1520 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.8031 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5519.9 | uss_MB: 4665.8 | child_rss_MB: 9015.1 | allocated_MB: 607.2 | reserved_MB: 31154.0 | inactive_split_MB: 255.6 +rank0 [2026-10-27 19:33:48,726] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 50% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 48.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 51.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:33:48,955] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/checkpoints/checkpoint-50000.pth +rank0 [2026-10-27 19:33:49,425] INFO : Running validation on rank0 +rank0 [2026-10-27 19:33:49,425] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 19:33:49,425] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 19:33:49,425] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 19:33:49,426] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 19:33:52,963] INFO : saved batch0_epoch50000.parquet +rank0 [2026-10-27 19:33:53,126] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 19:33:53,290] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 19:33:53,378] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 19:33:53,460] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 19:33:53,540] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 19:33:53,620] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 19:33:53,699] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 19:33:53,835] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 19:33:53,971] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 19:33:54,108] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 19:33:54,187] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 19:33:54,267] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 19:33:54,348] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 19:33:54,428] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 19:33:54,508] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 19:33:54,640] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 19:33:54,775] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 19:33:54,899] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 19:33:54,980] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 19:33:55,061] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 19:33:55,141] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 19:33:55,222] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 19:33:55,304] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 19:33:55,423] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 19:33:55,550] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 19:33:55,673] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 19:33:55,752] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 19:33:55,831] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 19:33:55,907] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 19:33:55,986] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 19:33:56,065] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 19:33:56,184] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 19:33:56,320] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 19:33:56,457] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 19:33:56,539] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 19:33:56,620] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 19:33:56,698] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 19:33:56,779] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 19:33:56,861] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 19:33:56,988] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 19:33:57,125] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 19:33:57,260] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 19:33:57,340] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 19:33:57,419] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 19:33:57,496] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 19:33:57,576] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 19:33:57,658] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 19:33:57,771] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 19:33:58,534] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 19:33:58,639] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 19:33:58,713] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 19:33:58,788] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 19:33:58,864] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 19:33:58,938] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 19:33:59,015] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 19:33:59,124] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 19:33:59,241] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 19:33:59,357] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 19:33:59,435] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 19:33:59,509] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 19:33:59,584] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 19:33:59,661] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 19:33:59,737] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 19:33:59,859] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 19:33:59,866] INFO : plotted pt_target histogram +rank0 [2026-10-27 19:33:59,869] INFO : plotted pt_pred histogram +rank0 [2026-10-27 19:33:59,872] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 19:33:59,875] INFO : plotted eta_target histogram +rank0 [2026-10-27 19:33:59,878] INFO : plotted eta_pred histogram +rank0 [2026-10-27 19:33:59,881] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 19:33:59,884] INFO : plotted sphi_target histogram +rank0 [2026-10-27 19:33:59,886] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 19:33:59,889] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 19:33:59,890] INFO : plotted cphi_target histogram +rank0 [2026-10-27 19:33:59,893] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 19:33:59,896] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 19:33:59,898] INFO : plotted energy_target histogram +rank0 [2026-10-27 19:33:59,901] INFO : plotted energy_pred histogram +rank0 [2026-10-27 19:33:59,904] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 19:35:46,542] INFO : VALIDATION | Step=50000/100000 | Train Loss=14.2027 | Valid Loss=12.7643 | Train Loss Unweighted=4.5299 | Valid Loss Unweighted=4.0606 | Group Valid Loss Unweighted: physical_pu: 3.6478 | physical_nopu: 4.4734 | Stale=0 | ETA=204.5m +rank0 [2026-10-27 19:35:46,552] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 19:35:46,704] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 19:35:46,707] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 19:36:12,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 19:36:12,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 19:36:13,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 19:36:13,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 19:36:14,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 19:36:14,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 19:36:15,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 19:36:15,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 19:36:25,842] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 19:36:25,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 19:36:26,396] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 19:36:26,399] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 19:36:41,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 19:36:42,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 19:36:42,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 19:36:43,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 19:36:43,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 19:36:44,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 19:36:44,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 19:36:45,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 19:36:55,538] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 19:36:55,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 19:36:55,957] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 19:36:55,961] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 19:37:11,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 19:37:11,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 19:37:12,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 19:37:12,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 19:37:13,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 19:37:13,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 19:37:14,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 19:37:15,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 19:37:25,103] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 19:37:25,159] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar +rank0 [2026-10-27 19:37:30,521] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar +rank0 [2026-10-27 19:37:31,796] INFO : Plotted number of elements +rank0 [2026-10-27 19:37:33,650] INFO : Plotted jets +rank0 [2026-10-27 19:37:34,956] INFO : Plotted jet ratio +rank0 [2026-10-27 19:37:35,000] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd +rank0 [2026-10-27 19:37:40,421] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd +rank0 [2026-10-27 19:37:41,640] INFO : Plotted number of elements +rank0 [2026-10-27 19:37:43,807] INFO : Plotted jets +rank0 [2026-10-27 19:37:44,917] INFO : Plotted jet ratio +rank0 [2026-10-27 19:37:44,955] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt +rank0 [2026-10-27 19:37:49,901] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt +rank0 [2026-10-27 19:37:51,106] INFO : Plotted number of elements +rank0 [2026-10-27 19:37:52,831] INFO : Plotted jets +rank0 [2026-10-27 19:37:54,010] INFO : Plotted jet ratio +rank0 [2026-10-27 19:41:36,410] INFO : Step 51000/100000 rank0 | Train Loss: 13.0997 | Train Loss Unweighted: 4.3733 | Train Loss Components: Classification: 0.0129 | Classification_binary: 3.5282 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3598 | Regression_eta: 0.0785 | Regression_pt: 0.2920 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6545 | physical_nopu: 4.4631 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3844 | Classification_binary: 3.5929 | Regression_cos_phi: 1.9037 | Regression_energy: 2.3967 | Regression_eta: 1.1442 | Regression_pt: 2.2492 | Regression_sin_phi: 2.2735 | LR: 2.14e-04 | DataLoad Time: 0.0111s | Model Forward Time: 3.7203s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1313 | max_loss: 0.0271 | max_backward: 0.1689 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.7590 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6858.3 | uss_MB: 6017.9 | child_rss_MB: 8958.7 | allocated_MB: 607.6 | reserved_MB: 29536.0 | inactive_split_MB: 388.7 +rank0 [2026-10-27 19:41:36,676] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 52.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:45:16,845] INFO : Step 52000/100000 rank0 | Train Loss: 11.7496 | Train Loss Unweighted: 3.6574 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9475 | Regression_cos_phi: 0.0469 | Regression_energy: 0.2717 | Regression_eta: 0.1056 | Regression_pt: 0.2231 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6567 | physical_nopu: 4.4685 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4605 | Classification_binary: 2.9661 | Regression_cos_phi: 1.7117 | Regression_energy: 1.4809 | Regression_eta: 1.5758 | Regression_pt: 1.6039 | Regression_sin_phi: 1.9325 | LR: 2.09e-04 | DataLoad Time: 0.0021s | Model Forward Time: 4.4631s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.1059 | max_forward: 0.2956 | max_loss: 0.0038 | max_backward: 0.4187 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 1.8177 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6858.3 | uss_MB: 6026.1 | child_rss_MB: 8942.2 | allocated_MB: 619.9 | reserved_MB: 29990.0 | inactive_split_MB: 1078.6 +rank0 [2026-10-27 19:45:17,098] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +2 6 0x7408, 15961 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 48.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 51.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:48:52,180] INFO : Step 53000/100000 rank0 | Train Loss: 12.5294 | Train Loss Unweighted: 4.3083 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5302 | Regression_cos_phi: 0.0494 | Regression_energy: 0.3242 | Regression_eta: 0.0722 | Regression_pt: 0.2705 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6530 | physical_nopu: 4.4629 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3717 | Classification_binary: 3.6354 | Regression_cos_phi: 2.2735 | Regression_energy: 1.6459 | Regression_eta: 1.3323 | Regression_pt: 1.8165 | Regression_sin_phi: 2.2470 | LR: 2.03e-04 | DataLoad Time: 0.0012s | Model Forward Time: 3.2372s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.1740 | max_loss: 0.0036 | max_backward: 0.1135 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 1.3124 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 5606.5 | uss_MB: 4766.5 | child_rss_MB: 8943.1 | allocated_MB: 608.5 | reserved_MB: 29990.0 | inactive_split_MB: 246.4 +rank0 [2026-10-27 19:48:52,434] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 1% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +2 6 0x7408, 15961 49.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 48.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 97% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 88% +6 10 0x7408, 62036 51.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:52:29,203] INFO : Step 54000/100000 rank0 | Train Loss: 13.6142 | Train Loss Unweighted: 4.3355 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.3742 | Regression_cos_phi: 0.0499 | Regression_energy: 0.4083 | Regression_eta: 0.0736 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4652 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4768 | Classification_binary: 3.4740 | Regression_cos_phi: 2.1113 | Regression_energy: 1.9640 | Regression_eta: 1.5824 | Regression_pt: 2.2304 | Regression_sin_phi: 2.2245 | LR: 1.97e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.1368s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1029 | max_loss: 0.0156 | max_backward: 0.1263 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2459 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 5600.5 | uss_MB: 4766.6 | child_rss_MB: 8886.2 | allocated_MB: 608.0 | reserved_MB: 29990.0 | inactive_split_MB: 249.1 +rank0 [2026-10-27 19:52:29,471] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 49.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 48.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 51.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:56:03,117] INFO : Step 55000/100000 rank0 | Train Loss: 14.4463 | Train Loss Unweighted: 4.6356 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5250 | Regression_cos_phi: 0.0497 | Regression_energy: 0.4959 | Regression_eta: 0.0660 | Regression_pt: 0.4403 | Regression_sin_phi: 0.0455 | Group Train Loss Unweighted (window mean): physical_pu: 3.6517 | physical_nopu: 4.4554 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3689 | Classification_binary: 3.3732 | Regression_cos_phi: 2.3295 | Regression_energy: 3.6273 | Regression_eta: 1.1909 | Regression_pt: 4.6397 | Regression_sin_phi: 1.8141 | LR: 1.92e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.6134s | Phase Times: max_pre_step_barrier: 0.0108 | max_batch_to_device: 0.0008 | max_forward: 0.0923 | max_loss: 0.0219 | max_backward: 0.1306 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 1.7796 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5599.8 | uss_MB: 4765.5 | child_rss_MB: 8883.4 | allocated_MB: 610.0 | reserved_MB: 29990.0 | inactive_split_MB: 274.0 +rank0 [2026-10-27 19:56:03,372] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 50.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 49.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 19:59:41,446] INFO : Step 56000/100000 rank0 | Train Loss: 12.4091 | Train Loss Unweighted: 4.2572 | Train Loss Components: Classification: 0.0135 | Classification_binary: 3.4523 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3421 | Regression_eta: 0.0664 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6493 | physical_nopu: 4.4547 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3657 | Classification_binary: 3.7699 | Regression_cos_phi: 2.1565 | Regression_energy: 2.1334 | Regression_eta: 1.2765 | Regression_pt: 2.2559 | Regression_sin_phi: 2.6127 | LR: 1.86e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.7145s | Phase Times: max_pre_step_barrier: 0.3290 | max_batch_to_device: 0.0902 | max_forward: 0.1771 | max_loss: 0.0038 | max_backward: 0.1058 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 1.4273 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5599.8 | uss_MB: 4766.9 | child_rss_MB: 8880.9 | allocated_MB: 608.1 | reserved_MB: 29990.0 | inactive_split_MB: 253.9 +rank0 [2026-10-27 19:59:41,733] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 50.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 49.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 53.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:03:30,233] INFO : Step 57000/100000 rank0 | Train Loss: 14.9087 | Train Loss Unweighted: 4.5957 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4371 | Regression_cos_phi: 0.0471 | Regression_energy: 0.5028 | Regression_eta: 0.0770 | Regression_pt: 0.4658 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6475 | physical_nopu: 4.4509 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6564 | Classification_binary: 3.2314 | Regression_cos_phi: 1.5736 | Regression_energy: 2.7601 | Regression_eta: 1.5800 | Regression_pt: 3.4347 | Regression_sin_phi: 2.0714 | LR: 1.81e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.2294s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1036 | max_loss: 0.0876 | max_backward: 0.2130 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 1.1951 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5597.8 | uss_MB: 4497.3 | child_rss_MB: 8866.5 | allocated_MB: 608.1 | reserved_MB: 30358.0 | inactive_split_MB: 283.9 +rank0 [2026-10-27 20:03:30,487] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 99% +2 6 0x7408, 15961 49.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 1% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 52.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:07:09,492] INFO : Step 58000/100000 rank0 | Train Loss: 12.4468 | Train Loss Unweighted: 4.2826 | Train Loss Components: Classification: 0.0105 | Classification_binary: 3.5299 | Regression_cos_phi: 0.0545 | Regression_energy: 0.3106 | Regression_eta: 0.0620 | Regression_pt: 0.2643 | Regression_sin_phi: 0.0509 | Group Train Loss Unweighted (window mean): physical_pu: 3.6440 | physical_nopu: 4.4492 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3862 | Classification_binary: 3.5805 | Regression_cos_phi: 2.0269 | Regression_energy: 2.1569 | Regression_eta: 0.9879 | Regression_pt: 2.3594 | Regression_sin_phi: 1.7487 | LR: 1.75e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.2845s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1868 | max_loss: 0.0037 | max_backward: 0.1111 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2860 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5587.6 | uss_MB: 3611.2 | child_rss_MB: 8803.1 | allocated_MB: 608.2 | reserved_MB: 30358.0 | inactive_split_MB: 337.0 +rank0 [2026-10-27 20:07:09,750] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 3% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 99% +2 6 0x7408, 15961 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +4 8 0x7408, 13395 48.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 52.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:10:53,128] INFO : Step 59000/100000 rank0 | Train Loss: 13.1870 | Train Loss Unweighted: 4.3659 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4760 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3771 | Regression_eta: 0.0773 | Regression_pt: 0.3247 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4440 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4870 | Classification_binary: 3.3492 | Regression_cos_phi: 1.6888 | Regression_energy: 2.4624 | Regression_eta: 1.2223 | Regression_pt: 3.0029 | Regression_sin_phi: 1.7267 | LR: 1.70e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.1178s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1005 | max_loss: 0.0036 | max_backward: 0.1062 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 1.2390 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 5582.6 | uss_MB: 4766.7 | child_rss_MB: 8755.9 | allocated_MB: 610.1 | reserved_MB: 31234.0 | inactive_split_MB: 273.9 +rank0 [2026-10-27 20:10:53,383] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 48.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 48.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 52.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:14:35,482] INFO : Step 60000/100000 rank0 | Train Loss: 15.4000 | Train Loss Unweighted: 4.6905 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.3905 | Regression_cos_phi: 0.0456 | Regression_energy: 0.5805 | Regression_eta: 0.0703 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6406 | physical_nopu: 4.4523 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4816 | Classification_binary: 3.2699 | Regression_cos_phi: 2.2178 | Regression_energy: 3.7551 | Regression_eta: 0.8974 | Regression_pt: 4.7953 | Regression_sin_phi: 2.0630 | LR: 1.64e-04 | DataLoad Time: 0.0014s | Model Forward Time: 3.2067s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0757 | max_forward: 0.1624 | max_loss: 0.0040 | max_backward: 0.1315 | max_optimizer_step: 0.2155 | max_tensorboard_logging: 1.2531 | max_tensorboard_flush: 0.0020 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5582.8 | uss_MB: 4766.6 | child_rss_MB: 8760.5 | allocated_MB: 611.6 | reserved_MB: 31234.0 | inactive_split_MB: 318.4 +rank0 [2026-10-27 20:14:35,733] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 48.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 48.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:14:35,902] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/checkpoints/checkpoint-60000.pth +rank0 [2026-10-27 20:14:36,470] INFO : Running validation on rank0 +rank0 [2026-10-27 20:14:36,471] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 20:14:36,471] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 20:14:36,471] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 20:14:36,471] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 20:14:39,683] INFO : saved batch0_epoch60000.parquet +rank0 [2026-10-27 20:14:39,841] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 20:14:40,000] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 20:14:40,085] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 20:14:40,163] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 20:14:40,240] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 20:14:40,317] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 20:14:40,396] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 20:14:40,526] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 20:14:40,660] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 20:14:40,795] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 20:14:40,874] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 20:14:40,952] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 20:14:41,030] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 20:14:41,108] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 20:14:41,187] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 20:14:41,314] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 20:14:41,437] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 20:14:41,557] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 20:14:41,635] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 20:14:41,712] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 20:14:41,789] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 20:14:41,867] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 20:14:41,945] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 20:14:42,054] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 20:14:42,175] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 20:14:42,294] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 20:14:42,371] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 20:14:42,451] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 20:14:42,529] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 20:14:42,610] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 20:14:42,689] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 20:14:42,817] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 20:14:42,955] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 20:14:43,090] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 20:14:43,170] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 20:14:43,253] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 20:14:43,332] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 20:14:43,414] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 20:14:43,498] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 20:14:43,626] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 20:14:43,765] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 20:14:43,898] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 20:14:43,979] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 20:14:44,060] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 20:14:44,142] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 20:14:44,224] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 20:14:44,997] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 20:14:45,130] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 20:14:45,242] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 20:14:45,350] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 20:14:45,428] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 20:14:45,506] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 20:14:45,583] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 20:14:45,662] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 20:14:45,741] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 20:14:45,849] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 20:14:45,966] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 20:14:46,080] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 20:14:46,158] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 20:14:46,238] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 20:14:46,316] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 20:14:46,395] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 20:14:46,472] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 20:14:46,585] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 20:14:46,591] INFO : plotted pt_target histogram +rank0 [2026-10-27 20:14:46,593] INFO : plotted pt_pred histogram +rank0 [2026-10-27 20:14:46,596] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 20:14:46,599] INFO : plotted eta_target histogram +rank0 [2026-10-27 20:14:46,602] INFO : plotted eta_pred histogram +rank0 [2026-10-27 20:14:46,605] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 20:14:46,608] INFO : plotted sphi_target histogram +rank0 [2026-10-27 20:14:46,610] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 20:14:46,613] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 20:14:46,616] INFO : plotted cphi_target histogram +rank0 [2026-10-27 20:14:46,617] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 20:14:46,619] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 20:14:46,622] INFO : plotted energy_target histogram +rank0 [2026-10-27 20:14:46,625] INFO : plotted energy_pred histogram +rank0 [2026-10-27 20:14:46,628] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 20:16:31,237] INFO : VALIDATION | Step=60000/100000 | Train Loss=15.4000 | Valid Loss=12.7136 | Train Loss Unweighted=4.6905 | Valid Loss Unweighted=4.0441 | Group Valid Loss Unweighted: physical_pu: 3.6377 | physical_nopu: 4.4504 | Stale=0 | ETA=163.5m +rank0 [2026-10-27 20:16:31,245] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 20:16:31,398] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 20:16:31,402] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 20:16:55,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 20:16:55,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 20:16:56,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 20:16:57,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 20:16:57,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 20:16:58,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 20:16:58,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 20:16:59,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 20:17:09,286] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 20:17:09,353] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 20:17:09,707] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 20:17:09,711] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 20:17:25,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 20:17:25,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 20:17:26,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 20:17:26,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 20:17:27,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 20:17:27,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 20:17:28,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 20:17:28,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 20:17:39,003] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 20:17:39,020] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 20:17:39,449] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 20:17:39,453] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 20:17:54,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 20:17:55,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 20:17:55,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 20:17:56,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 20:17:57,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 20:17:57,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 20:17:58,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 20:17:58,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 20:18:08,647] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 20:18:08,678] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar +rank0 [2026-10-27 20:18:13,600] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar +rank0 [2026-10-27 20:18:14,832] INFO : Plotted number of elements +rank0 [2026-10-27 20:18:16,628] INFO : Plotted jets +rank0 [2026-10-27 20:18:17,875] INFO : Plotted jet ratio +rank0 [2026-10-27 20:18:17,984] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd +rank0 [2026-10-27 20:18:23,399] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd +rank0 [2026-10-27 20:18:25,150] INFO : Plotted number of elements +rank0 [2026-10-27 20:18:26,838] INFO : Plotted jets +rank0 [2026-10-27 20:18:27,990] INFO : Plotted jet ratio +rank0 [2026-10-27 20:18:28,028] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt +rank0 [2026-10-27 20:18:32,955] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt +rank0 [2026-10-27 20:18:34,190] INFO : Plotted number of elements +rank0 [2026-10-27 20:18:35,945] INFO : Plotted jets +rank0 [2026-10-27 20:18:37,134] INFO : Plotted jet ratio +rank0 [2026-10-27 20:22:29,429] INFO : Step 61000/100000 rank0 | Train Loss: 12.1474 | Train Loss Unweighted: 4.3289 | Train Loss Components: Classification: 0.0110 | Classification_binary: 3.6236 | Regression_cos_phi: 0.0523 | Regression_energy: 0.2909 | Regression_eta: 0.0654 | Regression_pt: 0.2368 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.6358 | physical_nopu: 4.4439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3883 | Classification_binary: 3.7945 | Regression_cos_phi: 2.1954 | Regression_energy: 1.5839 | Regression_eta: 1.0272 | Regression_pt: 1.6405 | Regression_sin_phi: 2.5646 | LR: 1.59e-04 | DataLoad Time: 0.0076s | Model Forward Time: 4.1310s | Phase Times: max_pre_step_barrier: 0.0115 | max_batch_to_device: 0.0009 | max_forward: 0.2092 | max_loss: 0.0037 | max_backward: 0.1501 | max_optimizer_step: 0.2965 | max_tensorboard_logging: 1.8725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6817.2 | uss_MB: 6009.9 | child_rss_MB: 8755.0 | allocated_MB: 607.8 | reserved_MB: 28822.0 | inactive_split_MB: 222.9 +rank0 [2026-10-27 20:22:29,692] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 49.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:26:14,848] INFO : Step 62000/100000 rank0 | Train Loss: 11.7977 | Train Loss Unweighted: 3.6294 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9084 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2778 | Regression_eta: 0.1039 | Regression_pt: 0.2286 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6417 | physical_nopu: 4.4459 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4753 | Classification_binary: 2.9695 | Regression_cos_phi: 1.9357 | Regression_energy: 1.5026 | Regression_eta: 1.6694 | Regression_pt: 1.6063 | Regression_sin_phi: 1.8887 | LR: 1.54e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.3779s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0033 | max_forward: 0.2219 | max_loss: 0.0037 | max_backward: 0.2965 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1292 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1278 | Memory: rss_MB: 6812.4 | uss_MB: 6017.7 | child_rss_MB: 8760.3 | allocated_MB: 618.8 | reserved_MB: 28822.0 | inactive_split_MB: 888.0 +rank0 [2026-10-27 20:26:15,101] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 49.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 48.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:30:06,661] INFO : Step 63000/100000 rank0 | Train Loss: 12.5071 | Train Loss Unweighted: 4.3613 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5997 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3200 | Regression_eta: 0.0719 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6399 | physical_nopu: 4.4447 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3409 | Classification_binary: 3.6634 | Regression_cos_phi: 1.8690 | Regression_energy: 1.6645 | Regression_eta: 1.1906 | Regression_pt: 1.8047 | Regression_sin_phi: 1.8856 | LR: 1.49e-04 | DataLoad Time: 0.0022s | Model Forward Time: 11.8428s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1169 | max_loss: 0.0157 | max_backward: 0.2882 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 9.0944 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6806.2 | uss_MB: 4694.0 | child_rss_MB: 8747.4 | allocated_MB: 610.4 | reserved_MB: 28822.0 | inactive_split_MB: 261.6 +rank0 [2026-10-27 20:30:06,935] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 48.0°C 120.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 46% 0% +4 8 0x7408, 13395 47.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 51.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 40% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:33:54,079] INFO : Step 64000/100000 rank0 | Train Loss: 14.0388 | Train Loss Unweighted: 4.5035 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.4200 | Regression_cos_phi: 0.0430 | Regression_energy: 0.4752 | Regression_eta: 0.0663 | Regression_pt: 0.4376 | Regression_sin_phi: 0.0442 | Group Train Loss Unweighted (window mean): physical_pu: 3.6357 | physical_nopu: 4.4395 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4618 | Classification_binary: 3.3115 | Regression_cos_phi: 1.8413 | Regression_energy: 1.7185 | Regression_eta: 0.8587 | Regression_pt: 2.0042 | Regression_sin_phi: 1.6025 | LR: 1.43e-04 | DataLoad Time: 0.0024s | Model Forward Time: 7.1204s | Phase Times: max_pre_step_barrier: 0.1731 | max_batch_to_device: 0.0118 | max_forward: 0.1219 | max_loss: 0.0038 | max_backward: 0.1121 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 5.0441 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5600.0 | uss_MB: 4820.3 | child_rss_MB: 8744.3 | allocated_MB: 611.1 | reserved_MB: 29052.0 | inactive_split_MB: 372.9 +rank0 [2026-10-27 20:33:54,335] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 48.0°C 120.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 93% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +4 8 0x7408, 13395 47.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 10% +6 10 0x7408, 62036 51.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:37:49,239] INFO : Step 65000/100000 rank0 | Train Loss: 16.3374 | Train Loss Unweighted: 4.8919 | Train Loss Components: Classification: 0.0142 | Classification_binary: 3.4871 | Regression_cos_phi: 0.0500 | Regression_energy: 0.6360 | Regression_eta: 0.0628 | Regression_pt: 0.5943 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4335 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3664 | Classification_binary: 3.5721 | Regression_cos_phi: 2.1947 | Regression_energy: 2.0326 | Regression_eta: 1.2026 | Regression_pt: 2.3451 | Regression_sin_phi: 1.9690 | LR: 1.38e-04 | DataLoad Time: 0.0019s | Model Forward Time: 4.2085s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.2167 | max_loss: 0.0058 | max_backward: 0.2407 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 1.9060 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5594.0 | uss_MB: 4820.8 | child_rss_MB: 8750.8 | allocated_MB: 609.7 | reserved_MB: 29052.0 | inactive_split_MB: 236.9 +rank0 [2026-10-27 20:37:49,490] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 47.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:41:38,017] INFO : Step 66000/100000 rank0 | Train Loss: 12.8848 | Train Loss Unweighted: 4.2935 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4715 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3451 | Regression_eta: 0.0693 | Regression_pt: 0.2907 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.6349 | physical_nopu: 4.4348 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3880 | Classification_binary: 3.3976 | Regression_cos_phi: 1.7880 | Regression_energy: 1.8454 | Regression_eta: 1.4255 | Regression_pt: 1.9707 | Regression_sin_phi: 1.8234 | LR: 1.33e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.8176s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0854 | max_loss: 0.1083 | max_backward: 0.2178 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.9218 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5582.1 | uss_MB: 4820.7 | child_rss_MB: 8750.8 | allocated_MB: 611.4 | reserved_MB: 29052.0 | inactive_split_MB: 254.2 +rank0 [2026-10-27 20:41:38,275] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 49.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:45:25,147] INFO : Step 67000/100000 rank0 | Train Loss: 14.2125 | Train Loss Unweighted: 4.5026 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.4493 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4566 | Regression_eta: 0.0671 | Regression_pt: 0.4150 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6352 | physical_nopu: 4.4350 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4873 | Classification_binary: 3.2551 | Regression_cos_phi: 1.9730 | Regression_energy: 2.0674 | Regression_eta: 1.0169 | Regression_pt: 2.5067 | Regression_sin_phi: 1.9435 | LR: 1.28e-04 | DataLoad Time: 0.0029s | Model Forward Time: 3.6192s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0901 | max_loss: 0.0487 | max_backward: 0.1449 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 1.8090 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5575.1 | uss_MB: 4820.8 | child_rss_MB: 8750.2 | allocated_MB: 607.1 | reserved_MB: 29260.0 | inactive_split_MB: 246.6 +rank0 [2026-10-27 20:45:25,407] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 49.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:49:05,780] INFO : Step 68000/100000 rank0 | Train Loss: 12.0487 | Train Loss Unweighted: 4.2633 | Train Loss Components: Classification: 0.0107 | Classification_binary: 3.5135 | Regression_cos_phi: 0.0466 | Regression_energy: 0.3165 | Regression_eta: 0.0645 | Regression_pt: 0.2647 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4387 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3699 | Classification_binary: 3.4241 | Regression_cos_phi: 1.5101 | Regression_energy: 1.4579 | Regression_eta: 0.8254 | Regression_pt: 1.6248 | Regression_sin_phi: 1.8139 | LR: 1.24e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.5400s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0010 | max_forward: 0.0879 | max_loss: 0.0120 | max_backward: 0.1069 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.7176 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5570.9 | uss_MB: 4820.5 | child_rss_MB: 8753.3 | allocated_MB: 609.5 | reserved_MB: 29260.0 | inactive_split_MB: 225.9 +rank0 [2026-10-27 20:49:06,031] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:52:47,929] INFO : Step 69000/100000 rank0 | Train Loss: 13.1377 | Train Loss Unweighted: 4.3133 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4496 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3632 | Regression_eta: 0.0778 | Regression_pt: 0.3079 | Regression_sin_phi: 0.0541 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4321 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5495 | Classification_binary: 3.4702 | Regression_cos_phi: 1.8019 | Regression_energy: 2.1629 | Regression_eta: 0.9495 | Regression_pt: 2.3966 | Regression_sin_phi: 1.8758 | LR: 1.19e-04 | DataLoad Time: 0.0013s | Model Forward Time: 5.5556s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.1207 | max_loss: 0.0509 | max_backward: 0.2012 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 3.5532 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 5571.9 | uss_MB: 4820.8 | child_rss_MB: 8755.8 | allocated_MB: 608.0 | reserved_MB: 29260.0 | inactive_split_MB: 256.4 +rank0 [2026-10-27 20:52:48,185] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 123.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 48.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:56:28,131] INFO : Step 70000/100000 rank0 | Train Loss: 15.8477 | Train Loss Unweighted: 4.7317 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3873 | Regression_cos_phi: 0.0498 | Regression_energy: 0.6015 | Regression_eta: 0.0629 | Regression_pt: 0.5683 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6315 | physical_nopu: 4.4280 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5304 | Classification_binary: 3.3750 | Regression_cos_phi: 2.0560 | Regression_energy: 2.9963 | Regression_eta: 0.9187 | Regression_pt: 3.8896 | Regression_sin_phi: 1.7337 | LR: 1.14e-04 | DataLoad Time: 0.0047s | Model Forward Time: 3.5599s | Phase Times: max_pre_step_barrier: 0.0198 | max_batch_to_device: 0.0007 | max_forward: 0.1178 | max_loss: 0.0036 | max_backward: 0.1081 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.7610 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5571.5 | uss_MB: 4819.7 | child_rss_MB: 8751.0 | allocated_MB: 610.3 | reserved_MB: 30150.0 | inactive_split_MB: 260.3 +rank0 [2026-10-27 20:56:28,389] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 49.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +4 8 0x7408, 13395 48.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 51.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 20:56:28,614] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/checkpoints/checkpoint-70000.pth +rank0 [2026-10-27 20:56:29,212] INFO : Running validation on rank0 +rank0 [2026-10-27 20:56:29,212] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 20:56:29,212] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 20:56:29,212] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 20:56:29,212] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 20:56:32,972] INFO : saved batch0_epoch70000.parquet +rank0 [2026-10-27 20:56:33,136] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 20:56:33,293] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 20:56:33,378] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 20:56:33,459] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 20:56:33,540] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 20:56:33,622] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 20:56:33,705] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 20:56:33,843] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 20:56:33,979] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 20:56:34,113] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 20:56:34,195] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 20:56:34,274] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 20:56:34,356] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 20:56:34,439] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 20:56:34,520] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 20:56:34,649] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 20:56:34,778] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 20:56:34,903] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 20:56:34,983] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 20:56:35,065] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 20:56:35,145] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 20:56:35,226] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 20:56:35,305] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 20:56:35,421] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 20:56:35,548] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 20:56:35,669] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 20:56:35,747] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 20:56:35,826] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 20:56:35,904] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 20:56:35,982] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 20:56:36,061] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 20:56:36,186] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 20:56:36,327] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 20:56:36,459] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 20:56:36,539] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 20:56:36,618] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 20:56:36,698] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 20:56:36,775] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 20:56:36,856] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 20:56:36,975] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 20:56:37,114] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 20:56:37,246] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 20:56:37,325] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 20:56:37,402] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 20:56:37,479] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 20:56:37,558] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 20:56:37,637] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 20:56:38,447] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 20:56:38,558] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 20:56:38,667] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 20:56:38,746] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 20:56:38,826] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 20:56:38,904] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 20:56:38,985] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 20:56:39,063] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 20:56:39,174] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 20:56:39,290] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 20:56:39,407] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 20:56:39,485] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 20:56:39,563] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 20:56:39,640] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 20:56:39,722] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 20:56:39,802] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 20:56:39,912] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 20:56:39,986] INFO : plotted pt_target histogram +rank0 [2026-10-27 20:56:39,989] INFO : plotted pt_pred histogram +rank0 [2026-10-27 20:56:39,992] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 20:56:39,995] INFO : plotted eta_target histogram +rank0 [2026-10-27 20:56:39,998] INFO : plotted eta_pred histogram +rank0 [2026-10-27 20:56:40,001] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 20:56:40,002] INFO : plotted sphi_target histogram +rank0 [2026-10-27 20:56:40,005] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 20:56:40,007] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 20:56:40,010] INFO : plotted cphi_target histogram +rank0 [2026-10-27 20:56:40,013] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 20:56:40,015] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 20:56:40,018] INFO : plotted energy_target histogram +rank0 [2026-10-27 20:56:40,021] INFO : plotted energy_pred histogram +rank0 [2026-10-27 20:56:40,024] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 20:58:26,366] INFO : VALIDATION | Step=70000/100000 | Train Loss=15.8477 | Valid Loss=12.6754 | Train Loss Unweighted=4.7317 | Valid Loss Unweighted=4.0323 | Group Valid Loss Unweighted: physical_pu: 3.6311 | physical_nopu: 4.4334 | Stale=0 | ETA=123.1m +rank0 [2026-10-27 20:58:26,374] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 20:58:26,534] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 20:58:26,538] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 20:58:51,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 20:58:52,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 20:58:53,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 20:58:53,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 20:58:54,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 20:58:54,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 20:58:55,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 20:58:55,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 20:59:05,940] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 20:59:05,946] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 20:59:06,378] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 20:59:06,382] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 20:59:21,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 20:59:22,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 20:59:23,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 20:59:23,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 20:59:24,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 20:59:24,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 20:59:25,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 20:59:25,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 20:59:35,855] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 20:59:35,861] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 20:59:36,279] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 20:59:36,283] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 20:59:51,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 20:59:52,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 20:59:52,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 20:59:53,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 20:59:53,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 20:59:54,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 20:59:54,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 20:59:55,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 21:00:05,356] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 21:00:05,371] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar +rank0 [2026-10-27 21:00:10,381] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar +rank0 [2026-10-27 21:00:11,547] INFO : Plotted number of elements +rank0 [2026-10-27 21:00:13,287] INFO : Plotted jets +rank0 [2026-10-27 21:00:14,489] INFO : Plotted jet ratio +rank0 [2026-10-27 21:00:14,541] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd +rank0 [2026-10-27 21:00:19,761] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd +rank0 [2026-10-27 21:00:20,991] INFO : Plotted number of elements +rank0 [2026-10-27 21:00:23,215] INFO : Plotted jets +rank0 [2026-10-27 21:00:24,341] INFO : Plotted jet ratio +rank0 [2026-10-27 21:00:24,379] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt +rank0 [2026-10-27 21:00:29,319] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt +rank0 [2026-10-27 21:00:30,528] INFO : Plotted number of elements +rank0 [2026-10-27 21:00:32,269] INFO : Plotted jets +rank0 [2026-10-27 21:00:33,449] INFO : Plotted jet ratio +rank0 [2026-10-27 21:04:25,862] INFO : Step 71000/100000 rank0 | Train Loss: 12.0110 | Train Loss Unweighted: 3.6539 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9046 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2881 | Regression_eta: 0.1063 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6306 | physical_nopu: 4.4256 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4437 | Classification_binary: 2.9736 | Regression_cos_phi: 1.7431 | Regression_energy: 1.4317 | Regression_eta: 1.4642 | Regression_pt: 1.5564 | Regression_sin_phi: 1.8153 | LR: 1.10e-04 | DataLoad Time: 0.0019s | Model Forward Time: 4.2026s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0035 | max_forward: 0.1952 | max_loss: 0.0038 | max_backward: 0.3195 | max_optimizer_step: 0.3092 | max_tensorboard_logging: 1.7411 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6798.7 | uss_MB: 6058.3 | child_rss_MB: 8722.6 | allocated_MB: 623.5 | reserved_MB: 27422.0 | inactive_split_MB: 864.2 +rank0 [2026-10-27 21:04:26,125] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 48.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:08:02,622] INFO : Step 72000/100000 rank0 | Train Loss: 15.2878 | Train Loss Unweighted: 4.6447 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.3643 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5735 | Regression_eta: 0.0647 | Regression_pt: 0.5353 | Regression_sin_phi: 0.0449 | Group Train Loss Unweighted (window mean): physical_pu: 3.6275 | physical_nopu: 4.4239 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5768 | Classification_binary: 3.5407 | Regression_cos_phi: 1.9540 | Regression_energy: 1.9331 | Regression_eta: 1.0116 | Regression_pt: 2.3135 | Regression_sin_phi: 2.4530 | LR: 1.05e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.1719s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0232 | max_forward: 0.1174 | max_loss: 0.0036 | max_backward: 0.0921 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 1.2606 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5641.1 | uss_MB: 4896.7 | child_rss_MB: 8682.2 | allocated_MB: 607.6 | reserved_MB: 28586.0 | inactive_split_MB: 303.9 +rank0 [2026-10-27 21:08:02,875] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 49.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +4 8 0x7408, 13395 48.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:11:51,176] INFO : Step 73000/100000 rank0 | Train Loss: 12.7380 | Train Loss Unweighted: 4.2392 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.4842 | Regression_cos_phi: 0.0575 | Regression_energy: 0.3093 | Regression_eta: 0.0643 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.6309 | physical_nopu: 4.4270 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3239 | Classification_binary: 3.3414 | Regression_cos_phi: 2.1319 | Regression_energy: 1.6550 | Regression_eta: 0.8711 | Regression_pt: 1.9900 | Regression_sin_phi: 2.3515 | LR: 1.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.0644s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0985 | max_loss: 0.0632 | max_backward: 0.1654 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2139 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5637.1 | uss_MB: 4897.0 | child_rss_MB: 8681.3 | allocated_MB: 607.0 | reserved_MB: 28586.0 | inactive_split_MB: 227.5 +rank0 [2026-10-27 21:11:51,431] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 1% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +4 8 0x7408, 13395 49.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:15:34,897] INFO : Step 74000/100000 rank0 | Train Loss: 12.5735 | Train Loss Unweighted: 4.2478 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4076 | Regression_cos_phi: 0.0429 | Regression_energy: 0.3533 | Regression_eta: 0.0699 | Regression_pt: 0.3116 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6307 | physical_nopu: 4.4218 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4994 | Classification_binary: 3.1017 | Regression_cos_phi: 1.2894 | Regression_energy: 1.4465 | Regression_eta: 0.7076 | Regression_pt: 1.7979 | Regression_sin_phi: 1.4961 | LR: 9.68e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.1727s | Phase Times: max_pre_step_barrier: 0.1687 | max_batch_to_device: 0.0008 | max_forward: 0.0990 | max_loss: 0.0516 | max_backward: 0.1766 | max_optimizer_step: 0.1812 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5626.2 | uss_MB: 4897.3 | child_rss_MB: 8634.5 | allocated_MB: 608.6 | reserved_MB: 28586.0 | inactive_split_MB: 285.4 +rank0 [2026-10-27 21:15:35,156] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 3% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +2 6 0x7408, 15961 49.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:19:22,333] INFO : Step 75000/100000 rank0 | Train Loss: 15.1687 | Train Loss Unweighted: 4.6098 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.3751 | Regression_cos_phi: 0.0492 | Regression_energy: 0.5530 | Regression_eta: 0.0638 | Regression_pt: 0.5060 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4206 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4163 | Classification_binary: 3.0798 | Regression_cos_phi: 1.2774 | Regression_energy: 5.4473 | Regression_eta: 0.6831 | Regression_pt: 7.3822 | Regression_sin_phi: 1.4218 | LR: 9.27e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.1317s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1026 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.2820 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5619.2 | uss_MB: 4896.7 | child_rss_MB: 8602.7 | allocated_MB: 610.1 | reserved_MB: 28586.0 | inactive_split_MB: 259.7 +rank0 [2026-10-27 21:19:22,596] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:23:06,671] INFO : Step 76000/100000 rank0 | Train Loss: 12.2943 | Train Loss Unweighted: 4.2773 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5085 | Regression_cos_phi: 0.0476 | Regression_energy: 0.3262 | Regression_eta: 0.0732 | Regression_pt: 0.2631 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6289 | physical_nopu: 4.4209 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4320 | Classification_binary: 3.4985 | Regression_cos_phi: 1.7015 | Regression_energy: 1.8485 | Regression_eta: 0.9871 | Regression_pt: 2.0940 | Regression_sin_phi: 1.7211 | LR: 8.88e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.3094s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.1085 | max_optimizer_step: 0.1831 | max_tensorboard_logging: 1.4262 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 5617.3 | uss_MB: 4895.6 | child_rss_MB: 8601.6 | allocated_MB: 608.9 | reserved_MB: 28942.0 | inactive_split_MB: 227.1 +rank0 [2026-10-27 21:23:06,956] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 49.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 51.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:27:02,635] INFO : Step 77000/100000 rank0 | Train Loss: 13.8477 | Train Loss Unweighted: 4.4496 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4225 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4510 | Regression_eta: 0.0623 | Regression_pt: 0.4032 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6274 | physical_nopu: 4.4166 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5375 | Classification_binary: 3.4217 | Regression_cos_phi: 1.6940 | Regression_energy: 2.8483 | Regression_eta: 0.7919 | Regression_pt: 3.5960 | Regression_sin_phi: 1.9956 | LR: 8.50e-05 | DataLoad Time: 0.0018s | Model Forward Time: 6.7991s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0839 | max_forward: 0.1754 | max_loss: 0.0036 | max_backward: 0.1103 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 4.6772 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1374 | Memory: rss_MB: 5607.3 | uss_MB: 4838.5 | child_rss_MB: 8502.5 | allocated_MB: 611.0 | reserved_MB: 28942.0 | inactive_split_MB: 369.0 +rank0 [2026-10-27 21:27:02,889] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +2 6 0x7408, 15961 48.0°C 120.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 47.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:30:53,331] INFO : Step 78000/100000 rank0 | Train Loss: 14.1981 | Train Loss Unweighted: 4.6212 | Train Loss Components: Classification: 0.0133 | Classification_binary: 3.5747 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4573 | Regression_eta: 0.0669 | Regression_pt: 0.4114 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.6241 | physical_nopu: 4.4230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3571 | Classification_binary: 3.5511 | Regression_cos_phi: 1.9058 | Regression_energy: 2.0377 | Regression_eta: 0.9034 | Regression_pt: 2.4829 | Regression_sin_phi: 1.9614 | LR: 8.13e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.7643s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1149 | max_loss: 0.0636 | max_backward: 0.1914 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5607.3 | uss_MB: 4897.1 | child_rss_MB: 8495.1 | allocated_MB: 606.6 | reserved_MB: 28942.0 | inactive_split_MB: 309.2 +rank0 [2026-10-27 21:30:53,595] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 49.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 49.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:34:50,358] INFO : Step 79000/100000 rank0 | Train Loss: 12.3648 | Train Loss Unweighted: 4.2396 | Train Loss Components: Classification: 0.0145 | Classification_binary: 3.4636 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3226 | Regression_eta: 0.0628 | Regression_pt: 0.2792 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6254 | physical_nopu: 4.4183 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4466 | Classification_binary: 3.4818 | Regression_cos_phi: 1.7095 | Regression_energy: 1.6686 | Regression_eta: 1.1256 | Regression_pt: 2.0074 | Regression_sin_phi: 1.9209 | LR: 7.78e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.9735s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0936 | max_forward: 0.1936 | max_loss: 0.0037 | max_backward: 0.1134 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 2.0079 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 5602.4 | uss_MB: 4897.2 | child_rss_MB: 8483.1 | allocated_MB: 609.2 | reserved_MB: 29222.0 | inactive_split_MB: 324.6 +rank0 [2026-10-27 21:34:50,635] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 49.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 49.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +6 10 0x7408, 62036 53.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:38:37,974] INFO : Step 80000/100000 rank0 | Train Loss: 16.1833 | Train Loss Unweighted: 4.8046 | Train Loss Components: Classification: 0.0155 | Classification_binary: 3.4149 | Regression_cos_phi: 0.0453 | Regression_energy: 0.6189 | Regression_eta: 0.0801 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6233 | physical_nopu: 4.4142 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5327 | Classification_binary: 3.3339 | Regression_cos_phi: 1.3867 | Regression_energy: 3.0727 | Regression_eta: 2.0717 | Regression_pt: 3.8112 | Regression_sin_phi: 1.5622 | LR: 7.44e-05 | DataLoad Time: 0.0095s | Model Forward Time: 4.5754s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0008 | max_forward: 0.1071 | max_loss: 0.0037 | max_backward: 0.1380 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 2.6784 | max_tensorboard_flush: 0.0026 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 5596.4 | uss_MB: 4871.2 | child_rss_MB: 8436.2 | allocated_MB: 606.3 | reserved_MB: 29222.0 | inactive_split_MB: 471.7 +rank0 [2026-10-27 21:38:38,242] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 49.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +6 10 0x7408, 62036 52.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:38:38,488] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/checkpoints/checkpoint-80000.pth +rank0 [2026-10-27 21:38:39,061] INFO : Running validation on rank0 +rank0 [2026-10-27 21:38:39,061] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 21:38:39,061] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 21:38:39,061] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 21:38:39,061] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 21:38:42,990] INFO : saved batch0_epoch80000.parquet +rank0 [2026-10-27 21:38:43,162] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 21:38:43,323] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 21:38:43,407] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 21:38:43,489] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 21:38:43,569] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 21:38:43,650] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 21:38:43,730] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 21:38:43,863] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 21:38:44,000] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 21:38:44,135] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 21:38:44,215] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 21:38:44,296] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 21:38:44,375] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 21:38:44,456] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 21:38:44,537] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 21:38:44,669] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 21:38:44,800] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 21:38:44,924] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 21:38:45,004] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 21:38:45,084] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 21:38:45,162] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 21:38:45,241] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 21:38:45,319] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 21:38:45,433] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 21:38:45,563] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 21:38:45,682] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 21:38:45,760] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 21:38:45,837] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 21:38:45,912] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 21:38:45,988] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 21:38:46,066] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 21:38:46,186] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 21:38:46,326] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 21:38:46,458] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 21:38:46,538] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 21:38:46,617] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 21:38:46,693] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 21:38:46,771] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 21:38:46,847] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 21:38:46,963] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 21:38:47,099] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 21:38:47,235] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 21:38:47,317] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 21:38:47,394] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 21:38:47,470] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 21:38:47,546] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 21:38:47,625] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 21:38:48,437] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 21:38:48,549] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 21:38:48,659] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 21:38:48,735] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 21:38:48,813] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 21:38:48,892] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 21:38:48,969] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 21:38:49,047] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 21:38:49,158] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 21:38:49,275] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 21:38:49,391] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 21:38:49,467] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 21:38:49,546] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 21:38:49,623] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 21:38:49,702] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 21:38:49,778] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 21:38:49,888] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 21:38:49,970] INFO : plotted pt_target histogram +rank0 [2026-10-27 21:38:49,973] INFO : plotted pt_pred histogram +rank0 [2026-10-27 21:38:49,976] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 21:38:49,979] INFO : plotted eta_target histogram +rank0 [2026-10-27 21:38:49,982] INFO : plotted eta_pred histogram +rank0 [2026-10-27 21:38:49,985] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 21:38:49,988] INFO : plotted sphi_target histogram +rank0 [2026-10-27 21:38:49,991] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 21:38:49,994] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 21:38:49,997] INFO : plotted cphi_target histogram +rank0 [2026-10-27 21:38:49,999] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 21:38:50,002] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 21:38:50,005] INFO : plotted energy_target histogram +rank0 [2026-10-27 21:38:50,008] INFO : plotted energy_pred histogram +rank0 [2026-10-27 21:38:50,011] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 21:40:40,002] INFO : VALIDATION | Step=80000/100000 | Train Loss=16.1833 | Valid Loss=12.6420 | Train Loss Unweighted=4.8046 | Valid Loss Unweighted=4.0213 | Group Valid Loss Unweighted: physical_pu: 3.6232 | physical_nopu: 4.4194 | Stale=0 | ETA=82.3m +rank0 [2026-10-27 21:40:40,002] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 21:40:40,153] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 21:40:40,157] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 21:41:05,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 21:41:06,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 21:41:07,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 21:41:07,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 21:41:08,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 21:41:08,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 21:41:09,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 21:41:09,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 21:41:19,907] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 21:41:19,914] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 21:41:20,295] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 21:41:20,299] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 21:41:35,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 21:41:35,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 21:41:36,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 21:41:37,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 21:41:37,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 21:41:38,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 21:41:38,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 21:41:39,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 21:41:49,498] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 21:41:49,545] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 21:41:50,007] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 21:41:50,011] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 21:42:04,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 21:42:05,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 21:42:06,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 21:42:06,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 21:42:07,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 21:42:07,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 21:42:08,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 21:42:08,728] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 21:42:18,739] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 21:42:18,999] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar +rank0 [2026-10-27 21:42:24,112] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar +rank0 [2026-10-27 21:42:25,406] INFO : Plotted number of elements +rank0 [2026-10-27 21:42:27,211] INFO : Plotted jets +rank0 [2026-10-27 21:42:28,412] INFO : Plotted jet ratio +rank0 [2026-10-27 21:42:28,468] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd +rank0 [2026-10-27 21:42:33,746] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd +rank0 [2026-10-27 21:42:35,491] INFO : Plotted number of elements +rank0 [2026-10-27 21:42:37,154] INFO : Plotted jets +rank0 [2026-10-27 21:42:38,279] INFO : Plotted jet ratio +rank0 [2026-10-27 21:42:38,317] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt +rank0 [2026-10-27 21:42:43,361] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt +rank0 [2026-10-27 21:42:44,542] INFO : Plotted number of elements +rank0 [2026-10-27 21:42:46,321] INFO : Plotted jets +rank0 [2026-10-27 21:42:47,549] INFO : Plotted jet ratio +rank0 [2026-10-27 21:46:50,791] INFO : Step 81000/100000 rank0 | Train Loss: 11.7500 | Train Loss Unweighted: 3.6065 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.8934 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2712 | Regression_eta: 0.1074 | Regression_pt: 0.2244 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4130 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4735 | Classification_binary: 2.9113 | Regression_cos_phi: 1.8157 | Regression_energy: 1.4396 | Regression_eta: 1.6230 | Regression_pt: 1.5778 | Regression_sin_phi: 1.8853 | LR: 7.11e-05 | DataLoad Time: 0.0018s | Model Forward Time: 9.3346s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0975 | max_forward: 0.3237 | max_loss: 0.0036 | max_backward: 0.5496 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 6.3197 | max_tensorboard_flush: 0.0011 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6963.5 | uss_MB: 4835.8 | child_rss_MB: 8415.8 | allocated_MB: 619.9 | reserved_MB: 28598.0 | inactive_split_MB: 1192.4 +rank0 [2026-10-27 21:46:51,052] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +2 6 0x7408, 15961 48.0°C 118.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 47.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 50.0°C 120.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 35% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:50:34,498] INFO : Step 82000/100000 rank0 | Train Loss: 13.6704 | Train Loss Unweighted: 4.5604 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0419 | Regression_energy: 0.4618 | Regression_eta: 0.0599 | Regression_pt: 0.4291 | Regression_sin_phi: 0.0407 | Group Train Loss Unweighted (window mean): physical_pu: 3.6250 | physical_nopu: 4.4152 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3744 | Classification_binary: 3.6693 | Regression_cos_phi: 1.7556 | Regression_energy: 2.0763 | Regression_eta: 1.0637 | Regression_pt: 2.4138 | Regression_sin_phi: 1.9170 | LR: 6.80e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.7529s | Phase Times: max_pre_step_barrier: 0.1835 | max_batch_to_device: 0.0874 | max_forward: 0.1976 | max_loss: 0.0036 | max_backward: 0.1170 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 1.6260 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 5659.6 | uss_MB: 4976.9 | child_rss_MB: 8356.3 | allocated_MB: 607.5 | reserved_MB: 28598.0 | inactive_split_MB: 253.8 +rank0 [2026-10-27 21:50:34,755] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 48.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:54:19,867] INFO : Step 83000/100000 rank0 | Train Loss: 12.0270 | Train Loss Unweighted: 4.2414 | Train Loss Components: Classification: 0.0092 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0499 | Regression_energy: 0.3056 | Regression_eta: 0.0671 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4141 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3285 | Classification_binary: 3.7094 | Regression_cos_phi: 1.7938 | Regression_energy: 1.3614 | Regression_eta: 1.1585 | Regression_pt: 1.3306 | Regression_sin_phi: 1.9340 | LR: 6.51e-05 | DataLoad Time: 0.0023s | Model Forward Time: 4.3428s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1966 | max_loss: 0.0138 | max_backward: 0.1124 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 2.4026 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 5659.6 | uss_MB: 4977.1 | child_rss_MB: 8350.2 | allocated_MB: 609.6 | reserved_MB: 28598.0 | inactive_split_MB: 289.5 +rank0 [2026-10-27 21:54:20,120] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 48.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 21:58:09,172] INFO : Step 84000/100000 rank0 | Train Loss: 12.5172 | Train Loss Unweighted: 4.2412 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.4173 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3464 | Regression_eta: 0.0637 | Regression_pt: 0.3052 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4149 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.7012 | Classification_binary: 3.2641 | Regression_cos_phi: 1.9020 | Regression_energy: 1.9581 | Regression_eta: 0.9429 | Regression_pt: 2.4392 | Regression_sin_phi: 1.7599 | LR: 6.23e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.9070s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1103 | max_loss: 0.0037 | max_backward: 0.1159 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 2.0319 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5655.6 | uss_MB: 4977.8 | child_rss_MB: 8312.0 | allocated_MB: 605.9 | reserved_MB: 28598.0 | inactive_split_MB: 208.8 +rank0 [2026-10-27 21:58:09,424] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 49.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 47.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:01:47,239] INFO : Step 85000/100000 rank0 | Train Loss: 15.2814 | Train Loss Unweighted: 4.6863 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.3811 | Regression_cos_phi: 0.0432 | Regression_energy: 0.5842 | Regression_eta: 0.0608 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6173 | physical_nopu: 4.4078 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4940 | Classification_binary: 3.2728 | Regression_cos_phi: 1.2956 | Regression_energy: 2.4972 | Regression_eta: 0.8578 | Regression_pt: 3.2546 | Regression_sin_phi: 1.3625 | LR: 5.96e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.7584s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.0993 | max_loss: 0.0558 | max_backward: 0.1762 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 1.8022 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 5645.3 | uss_MB: 4977.4 | child_rss_MB: 8242.1 | allocated_MB: 606.6 | reserved_MB: 28598.0 | inactive_split_MB: 257.4 +rank0 [2026-10-27 22:01:47,493] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 49.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:05:29,354] INFO : Step 86000/100000 rank0 | Train Loss: 12.8074 | Train Loss Unweighted: 4.3373 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.5785 | Regression_cos_phi: 0.0543 | Regression_energy: 0.3116 | Regression_eta: 0.0747 | Regression_pt: 0.2515 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.6255 | physical_nopu: 4.4086 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4233 | Classification_binary: 3.7965 | Regression_cos_phi: 2.0930 | Regression_energy: 2.0913 | Regression_eta: 0.9492 | Regression_pt: 2.2690 | Regression_sin_phi: 2.4597 | LR: 5.71e-05 | DataLoad Time: 0.0021s | Model Forward Time: 3.6939s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1215 | max_loss: 0.0037 | max_backward: 0.1229 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 1.6761 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5636.3 | uss_MB: 4977.9 | child_rss_MB: 8180.8 | allocated_MB: 608.3 | reserved_MB: 29688.0 | inactive_split_MB: 271.8 +rank0 [2026-10-27 22:05:29,609] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:09:23,277] INFO : Step 87000/100000 rank0 | Train Loss: 12.9093 | Train Loss Unweighted: 4.3003 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.3807 | Regression_cos_phi: 0.0447 | Regression_energy: 0.4014 | Regression_eta: 0.0615 | Regression_pt: 0.3523 | Regression_sin_phi: 0.0424 | Group Train Loss Unweighted (window mean): physical_pu: 3.6211 | physical_nopu: 4.4120 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6071 | Classification_binary: 3.3334 | Regression_cos_phi: 1.6578 | Regression_energy: 1.8958 | Regression_eta: 1.1722 | Regression_pt: 2.3172 | Regression_sin_phi: 1.6767 | LR: 5.48e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.7284s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0203 | max_forward: 0.1299 | max_loss: 0.0038 | max_backward: 0.1175 | max_optimizer_step: 0.2192 | max_tensorboard_logging: 1.8010 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5636.1 | uss_MB: 4575.9 | child_rss_MB: 8182.7 | allocated_MB: 607.0 | reserved_MB: 29688.0 | inactive_split_MB: 231.0 +rank0 [2026-10-27 22:09:23,534] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 48.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 47.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:13:03,488] INFO : Step 88000/100000 rank0 | Train Loss: 15.2369 | Train Loss Unweighted: 4.6456 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4525 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5240 | Regression_eta: 0.0665 | Regression_pt: 0.4840 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6192 | physical_nopu: 4.4119 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5182 | Classification_binary: 3.3685 | Regression_cos_phi: 2.3394 | Regression_energy: 1.7526 | Regression_eta: 0.8733 | Regression_pt: 1.8869 | Regression_sin_phi: 1.6721 | LR: 5.26e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.7994s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0007 | max_forward: 0.1127 | max_loss: 0.0236 | max_backward: 0.1542 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 1.7956 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5636.1 | uss_MB: 4977.3 | child_rss_MB: 8181.7 | allocated_MB: 609.0 | reserved_MB: 29688.0 | inactive_split_MB: 221.0 +rank0 [2026-10-27 22:13:03,763] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 132.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +2 6 0x7408, 15961 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 51.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:16:53,609] INFO : Step 89000/100000 rank0 | Train Loss: 12.8609 | Train Loss Unweighted: 4.2765 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.4665 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3343 | Regression_eta: 0.0681 | Regression_pt: 0.2896 | Regression_sin_phi: 0.0555 | Group Train Loss Unweighted (window mean): physical_pu: 3.6177 | physical_nopu: 4.4110 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4400 | Classification_binary: 3.3686 | Regression_cos_phi: 1.9230 | Regression_energy: 1.6152 | Regression_eta: 0.8872 | Regression_pt: 1.9899 | Regression_sin_phi: 2.3719 | LR: 5.06e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.8488s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1194 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 1.9452 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 5632.5 | uss_MB: 4977.7 | child_rss_MB: 8137.7 | allocated_MB: 612.6 | reserved_MB: 30628.0 | inactive_split_MB: 309.7 +rank0 [2026-10-27 22:16:53,862] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:20:44,424] INFO : Step 90000/100000 rank0 | Train Loss: 11.6102 | Train Loss Unweighted: 3.6127 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9177 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2661 | Regression_eta: 0.1040 | Regression_pt: 0.2155 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4109 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4784 | Classification_binary: 2.9039 | Regression_cos_phi: 1.8619 | Regression_energy: 1.4645 | Regression_eta: 1.5699 | Regression_pt: 1.5750 | Regression_sin_phi: 1.8957 | LR: 4.88e-05 | DataLoad Time: 0.0023s | Model Forward Time: 4.2867s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0110 | max_forward: 0.1856 | max_loss: 0.0145 | max_backward: 0.4526 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 1.8620 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 5624.5 | uss_MB: 4985.7 | child_rss_MB: 8062.4 | allocated_MB: 617.2 | reserved_MB: 30628.0 | inactive_split_MB: 962.2 +rank0 [2026-10-27 22:20:44,688] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 48.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:20:44,832] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/checkpoints/checkpoint-90000.pth +rank0 [2026-10-27 22:20:45,405] INFO : Running validation on rank0 +rank0 [2026-10-27 22:20:45,405] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 22:20:45,405] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 22:20:45,405] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 22:20:45,405] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 22:20:49,202] INFO : saved batch0_epoch90000.parquet +rank0 [2026-10-27 22:20:49,366] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 22:20:49,530] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 22:20:49,616] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 22:20:49,699] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 22:20:49,781] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 22:20:49,865] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 22:20:49,948] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 22:20:50,086] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 22:20:50,225] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 22:20:50,364] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 22:20:50,444] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 22:20:50,526] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 22:20:50,608] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 22:20:50,691] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 22:20:50,773] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 22:20:50,915] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 22:20:51,040] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 22:20:51,164] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 22:20:51,246] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 22:20:51,329] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 22:20:51,411] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 22:20:51,495] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 22:20:51,577] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 22:20:51,696] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 22:20:51,825] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 22:20:51,948] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 22:20:52,028] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 22:20:52,111] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 22:20:52,190] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 22:20:52,269] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 22:20:52,348] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 22:20:52,469] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 22:20:52,613] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 22:20:52,749] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 22:20:52,829] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 22:20:52,912] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 22:20:52,990] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 22:20:53,067] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 22:20:53,147] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 22:20:53,266] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 22:20:53,408] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 22:20:53,542] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 22:20:53,620] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 22:20:53,699] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 22:20:53,776] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 22:20:53,855] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 22:20:53,934] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 22:20:54,069] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 22:20:54,852] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 22:20:54,963] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 22:20:55,040] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 22:20:55,116] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 22:20:55,194] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 22:20:55,272] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 22:20:55,350] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 22:20:55,460] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 22:20:55,577] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 22:20:55,692] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 22:20:55,769] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 22:20:55,846] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 22:20:55,923] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 22:20:56,004] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 22:20:56,082] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 22:20:56,196] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 22:20:56,212] INFO : plotted pt_target histogram +rank0 [2026-10-27 22:20:56,224] INFO : plotted pt_pred histogram +rank0 [2026-10-27 22:20:56,227] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 22:20:56,229] INFO : plotted eta_target histogram +rank0 [2026-10-27 22:20:56,240] INFO : plotted eta_pred histogram +rank0 [2026-10-27 22:20:56,243] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 22:20:56,256] INFO : plotted sphi_target histogram +rank0 [2026-10-27 22:20:56,259] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 22:20:56,381] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 22:20:56,389] INFO : plotted cphi_target histogram +rank0 [2026-10-27 22:20:56,393] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 22:20:56,396] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 22:20:56,399] INFO : plotted energy_target histogram +rank0 [2026-10-27 22:20:56,402] INFO : plotted energy_pred histogram +rank0 [2026-10-27 22:20:56,408] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 22:22:45,630] INFO : VALIDATION | Step=90000/100000 | Train Loss=11.6102 | Valid Loss=12.6245 | Train Loss Unweighted=3.6127 | Valid Loss Unweighted=4.0154 | Group Valid Loss Unweighted: physical_pu: 3.6196 | physical_nopu: 4.4113 | Stale=0 | ETA=41.3m +rank0 [2026-10-27 22:22:45,638] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 22:22:45,789] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 22:22:45,793] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 22:23:10,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 22:23:10,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 22:23:11,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 22:23:12,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 22:23:12,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 22:23:13,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 22:23:13,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 22:23:14,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 22:23:24,286] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 22:23:24,364] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 22:23:24,733] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 22:23:24,737] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 22:23:39,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 22:23:40,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 22:23:41,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 22:23:41,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 22:23:42,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 22:23:42,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 22:23:43,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 22:23:43,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 22:23:53,859] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 22:23:53,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 22:23:54,276] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 22:23:54,280] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 22:24:09,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 22:24:10,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 22:24:10,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 22:24:11,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 22:24:11,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 22:24:12,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 22:24:12,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 22:24:13,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 22:24:23,386] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 22:24:23,399] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar +rank0 [2026-10-27 22:24:28,457] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar +rank0 [2026-10-27 22:24:29,711] INFO : Plotted number of elements +rank0 [2026-10-27 22:24:31,449] INFO : Plotted jets +rank0 [2026-10-27 22:24:32,656] INFO : Plotted jet ratio +rank0 [2026-10-27 22:24:32,708] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd +rank0 [2026-10-27 22:24:38,068] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd +rank0 [2026-10-27 22:24:39,422] INFO : Plotted number of elements +rank0 [2026-10-27 22:24:41,627] INFO : Plotted jets +rank0 [2026-10-27 22:24:42,747] INFO : Plotted jet ratio +rank0 [2026-10-27 22:24:42,785] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt +rank0 [2026-10-27 22:24:47,800] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt +rank0 [2026-10-27 22:24:49,054] INFO : Plotted number of elements +rank0 [2026-10-27 22:24:50,788] INFO : Plotted jets +rank0 [2026-10-27 22:24:51,992] INFO : Plotted jet ratio +rank0 [2026-10-27 22:28:41,836] INFO : Step 91000/100000 rank0 | Train Loss: 12.9063 | Train Loss Unweighted: 4.3371 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5236 | Regression_cos_phi: 0.0531 | Regression_energy: 0.3431 | Regression_eta: 0.0707 | Regression_pt: 0.2829 | Regression_sin_phi: 0.0512 | Group Train Loss Unweighted (window mean): physical_pu: 3.6180 | physical_nopu: 4.4087 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3456 | Classification_binary: 3.6668 | Regression_cos_phi: 2.0767 | Regression_energy: 1.6763 | Regression_eta: 0.9723 | Regression_pt: 1.9048 | Regression_sin_phi: 1.7297 | LR: 4.71e-05 | DataLoad Time: 0.0017s | Model Forward Time: 5.5223s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0055 | max_forward: 0.1521 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 3.4850 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7066.6 | uss_MB: 6432.9 | child_rss_MB: 8020.4 | allocated_MB: 607.8 | reserved_MB: 28796.0 | inactive_split_MB: 320.2 +rank0 [2026-10-27 22:28:42,093] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 48.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:32:24,224] INFO : Step 92000/100000 rank0 | Train Loss: 15.4341 | Train Loss Unweighted: 4.7247 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4285 | Regression_cos_phi: 0.0457 | Regression_energy: 0.5811 | Regression_eta: 0.0735 | Regression_pt: 0.5375 | Regression_sin_phi: 0.0425 | Group Train Loss Unweighted (window mean): physical_pu: 3.6150 | physical_nopu: 4.4032 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4404 | Classification_binary: 3.5658 | Regression_cos_phi: 1.6940 | Regression_energy: 2.7259 | Regression_eta: 1.0326 | Regression_pt: 3.1622 | Regression_sin_phi: 1.7988 | LR: 4.57e-05 | DataLoad Time: 0.0019s | Model Forward Time: 4.1847s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.2696 | max_loss: 0.0037 | max_backward: 0.1715 | max_optimizer_step: 0.3237 | max_tensorboard_logging: 1.7768 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7051.6 | uss_MB: 6432.9 | child_rss_MB: 7931.4 | allocated_MB: 609.3 | reserved_MB: 28796.0 | inactive_split_MB: 222.8 +rank0 [2026-10-27 22:32:24,483] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 21% +2 6 0x7408, 15961 48.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 52.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:36:09,438] INFO : Step 93000/100000 rank0 | Train Loss: 13.9744 | Train Loss Unweighted: 4.4854 | Train Loss Components: Classification: 0.0118 | Classification_binary: 3.5124 | Regression_cos_phi: 0.0578 | Regression_energy: 0.4245 | Regression_eta: 0.0652 | Regression_pt: 0.3624 | Regression_sin_phi: 0.0513 | Group Train Loss Unweighted (window mean): physical_pu: 3.6205 | physical_nopu: 4.4075 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4054 | Classification_binary: 3.4986 | Regression_cos_phi: 1.4829 | Regression_energy: 2.4381 | Regression_eta: 1.1251 | Regression_pt: 2.9465 | Regression_sin_phi: 2.0413 | LR: 4.43e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.6238s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1098 | max_loss: 0.0036 | max_backward: 0.1173 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 1.7709 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5706.9 | uss_MB: 5092.9 | child_rss_MB: 7894.0 | allocated_MB: 606.4 | reserved_MB: 30394.0 | inactive_split_MB: 244.2 +rank0 [2026-10-27 22:36:09,698] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:39:53,734] INFO : Step 94000/100000 rank0 | Train Loss: 12.4397 | Train Loss Unweighted: 4.2538 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4230 | Regression_cos_phi: 0.0415 | Regression_energy: 0.3513 | Regression_eta: 0.0702 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6164 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4720 | Classification_binary: 3.3446 | Regression_cos_phi: 1.4585 | Regression_energy: 1.7042 | Regression_eta: 0.8840 | Regression_pt: 1.9897 | Regression_sin_phi: 1.8264 | LR: 4.32e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.8459s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0009 | max_forward: 0.1540 | max_loss: 0.0036 | max_backward: 0.2309 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 1.7723 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 5695.9 | uss_MB: 4753.0 | child_rss_MB: 7806.9 | allocated_MB: 607.1 | reserved_MB: 30394.0 | inactive_split_MB: 232.2 +rank0 [2026-10-27 22:39:53,995] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 49.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 51.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:43:37,614] INFO : Step 95000/100000 rank0 | Train Loss: 14.9177 | Train Loss Unweighted: 4.6108 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3911 | Regression_cos_phi: 0.0436 | Regression_energy: 0.5612 | Regression_eta: 0.0598 | Regression_pt: 0.4905 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6121 | physical_nopu: 4.4076 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4470 | Classification_binary: 3.4487 | Regression_cos_phi: 2.0750 | Regression_energy: 2.1940 | Regression_eta: 1.0427 | Regression_pt: 2.5042 | Regression_sin_phi: 2.2630 | LR: 4.22e-05 | DataLoad Time: 0.0010s | Model Forward Time: 3.6934s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1111 | max_loss: 0.0199 | max_backward: 0.1395 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.7978 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5695.9 | uss_MB: 5093.1 | child_rss_MB: 7811.5 | allocated_MB: 609.0 | reserved_MB: 30394.0 | inactive_split_MB: 285.3 +rank0 [2026-10-27 22:43:37,870] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 52.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:47:15,088] INFO : Step 96000/100000 rank0 | Train Loss: 12.4738 | Train Loss Unweighted: 4.2244 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.4884 | Regression_cos_phi: 0.0540 | Regression_energy: 0.2967 | Regression_eta: 0.0804 | Regression_pt: 0.2429 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6137 | physical_nopu: 4.4081 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2877 | Classification_binary: 3.4257 | Regression_cos_phi: 2.0159 | Regression_energy: 1.3089 | Regression_eta: 1.1950 | Regression_pt: 1.3763 | Regression_sin_phi: 1.8399 | LR: 4.14e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.8903s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0007 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.2003 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 1.8675 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5695.9 | uss_MB: 5093.2 | child_rss_MB: 7804.9 | allocated_MB: 608.1 | reserved_MB: 30394.0 | inactive_split_MB: 259.4 +rank0 [2026-10-27 22:47:15,349] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 1% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +2 6 0x7408, 15961 48.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 77% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +6 10 0x7408, 62036 51.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:50:53,665] INFO : Step 97000/100000 rank0 | Train Loss: 12.7922 | Train Loss Unweighted: 4.2852 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.4355 | Regression_cos_phi: 0.0455 | Regression_energy: 0.3604 | Regression_eta: 0.0703 | Regression_pt: 0.3091 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6136 | physical_nopu: 4.4015 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4847 | Classification_binary: 3.5817 | Regression_cos_phi: 1.7177 | Regression_energy: 1.5675 | Regression_eta: 1.0637 | Regression_pt: 1.8870 | Regression_sin_phi: 2.0680 | LR: 4.08e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.6387s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1110 | max_loss: 0.0174 | max_backward: 0.1239 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 1.7282 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0908 | Memory: rss_MB: 5687.4 | uss_MB: 5092.4 | child_rss_MB: 7746.9 | allocated_MB: 607.2 | reserved_MB: 30394.0 | inactive_split_MB: 265.3 +rank0 [2026-10-27 22:50:53,928] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 132.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 52.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:54:50,286] INFO : Step 98000/100000 rank0 | Train Loss: 13.8508 | Train Loss Unweighted: 4.4512 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3736 | Regression_cos_phi: 0.0443 | Regression_energy: 0.4792 | Regression_eta: 0.0606 | Regression_pt: 0.4354 | Regression_sin_phi: 0.0433 | Group Train Loss Unweighted (window mean): physical_pu: 3.6169 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4562 | Classification_binary: 3.3407 | Regression_cos_phi: 1.4602 | Regression_energy: 2.6923 | Regression_eta: 0.8178 | Regression_pt: 3.3908 | Regression_sin_phi: 1.5773 | LR: 4.04e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.7293s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0119 | max_forward: 0.1317 | max_loss: 0.0036 | max_backward: 0.1246 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.7312 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0878 | Memory: rss_MB: 5687.4 | uss_MB: 5093.6 | child_rss_MB: 7731.7 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 292.7 +rank0 [2026-10-27 22:54:50,543] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 49.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 48.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 52.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 22:58:30,776] INFO : Step 99000/100000 rank0 | Train Loss: 12.5729 | Train Loss Unweighted: 4.3128 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5536 | Regression_cos_phi: 0.0519 | Regression_energy: 0.3097 | Regression_eta: 0.0644 | Regression_pt: 0.2679 | Regression_sin_phi: 0.0529 | Group Train Loss Unweighted (window mean): physical_pu: 3.6154 | physical_nopu: 4.4035 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4467 | Classification_binary: 3.5335 | Regression_cos_phi: 1.8456 | Regression_energy: 1.5575 | Regression_eta: 0.8922 | Regression_pt: 1.8259 | Regression_sin_phi: 1.8041 | LR: 4.01e-05 | DataLoad Time: 0.0102s | Model Forward Time: 3.6688s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0008 | max_forward: 0.1158 | max_loss: 0.0113 | max_backward: 0.1168 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 1.7726 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 5687.4 | uss_MB: 5093.5 | child_rss_MB: 7727.1 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 283.7 +rank0 [2026-10-27 22:58:31,043] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 23:02:17,811] INFO : Step 100000/100000 rank0 | Train Loss: 11.9488 | Train Loss Unweighted: 3.6450 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9057 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2836 | Regression_eta: 0.1038 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6201 | physical_nopu: 4.4070 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4871 | Classification_binary: 2.9065 | Regression_cos_phi: 1.7442 | Regression_energy: 1.4529 | Regression_eta: 1.5462 | Regression_pt: 1.6417 | Regression_sin_phi: 1.9005 | LR: 4.00e-05 | DataLoad Time: 0.0030s | Model Forward Time: 7.0890s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0031 | max_forward: 0.2860 | max_loss: 0.0036 | max_backward: 0.4897 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 4.4601 | max_tensorboard_flush: 0.0009 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5687.2 | uss_MB: 4556.1 | child_rss_MB: 7711.3 | allocated_MB: 618.6 | reserved_MB: 30394.0 | inactive_split_MB: 1112.1 +rank0 [2026-10-27 23:02:18,073] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +2 6 0x7408, 15961 48.0°C 120.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 72% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 48.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +6 10 0x7408, 62036 52.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 89% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-27 23:02:18,114] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/checkpoints/checkpoint-100000.pth +rank0 [2026-10-27 23:02:18,787] INFO : Running validation on rank0 +rank0 [2026-10-27 23:02:18,787] INFO : ResumableSampler physical_pu:valid resetting +rank0 [2026-10-27 23:02:18,787] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank0 [2026-10-27 23:02:18,787] INFO : ResumableSampler physical_nopu:valid resetting +rank0 [2026-10-27 23:02:18,787] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank0 [2026-10-27 23:02:22,615] INFO : saved batch0_epoch100000.parquet +rank0 [2026-10-27 23:02:22,778] INFO : plotted energy_elemtype1 +rank0 [2026-10-27 23:02:22,946] INFO : plotted pt_elemtype1 +rank0 [2026-10-27 23:02:23,033] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-27 23:02:23,117] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-27 23:02:23,200] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-27 23:02:23,281] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-27 23:02:23,362] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-27 23:02:23,498] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-27 23:02:23,642] INFO : plotted energy_elemtype4 +rank0 [2026-10-27 23:02:23,781] INFO : plotted pt_elemtype4 +rank0 [2026-10-27 23:02:23,863] INFO : plotted energy_elemtype4_corr +rank0 [2026-10-27 23:02:23,945] INFO : plotted pt_elemtype4_corr +rank0 [2026-10-27 23:02:24,026] INFO : plotted eta_elemtype4_corr +rank0 [2026-10-27 23:02:24,108] INFO : plotted sphi_elemtype4_corr +rank0 [2026-10-27 23:02:24,190] INFO : plotted cphi_elemtype4_corr +rank0 [2026-10-27 23:02:24,328] INFO : plotted sig_proba_elemtype4 +rank0 [2026-10-27 23:02:24,465] INFO : plotted energy_elemtype5 +rank0 [2026-10-27 23:02:24,591] INFO : plotted pt_elemtype5 +rank0 [2026-10-27 23:02:24,673] INFO : plotted energy_elemtype5_corr +rank0 [2026-10-27 23:02:24,755] INFO : plotted pt_elemtype5_corr +rank0 [2026-10-27 23:02:24,840] INFO : plotted eta_elemtype5_corr +rank0 [2026-10-27 23:02:24,922] INFO : plotted sphi_elemtype5_corr +rank0 [2026-10-27 23:02:25,005] INFO : plotted cphi_elemtype5_corr +rank0 [2026-10-27 23:02:25,120] INFO : plotted sig_proba_elemtype5 +rank0 [2026-10-27 23:02:25,234] INFO : plotted energy_elemtype6 +rank0 [2026-10-27 23:02:25,345] INFO : plotted pt_elemtype6 +rank0 [2026-10-27 23:02:25,425] INFO : plotted energy_elemtype6_corr +rank0 [2026-10-27 23:02:25,508] INFO : plotted pt_elemtype6_corr +rank0 [2026-10-27 23:02:25,588] INFO : plotted eta_elemtype6_corr +rank0 [2026-10-27 23:02:25,668] INFO : plotted sphi_elemtype6_corr +rank0 [2026-10-27 23:02:25,749] INFO : plotted cphi_elemtype6_corr +rank0 [2026-10-27 23:02:25,871] INFO : plotted sig_proba_elemtype6 +rank0 [2026-10-27 23:02:26,009] INFO : plotted energy_elemtype8 +rank0 [2026-10-27 23:02:26,148] INFO : plotted pt_elemtype8 +rank0 [2026-10-27 23:02:26,229] INFO : plotted energy_elemtype8_corr +rank0 [2026-10-27 23:02:26,310] INFO : plotted pt_elemtype8_corr +rank0 [2026-10-27 23:02:26,388] INFO : plotted eta_elemtype8_corr +rank0 [2026-10-27 23:02:26,467] INFO : plotted sphi_elemtype8_corr +rank0 [2026-10-27 23:02:26,547] INFO : plotted cphi_elemtype8_corr +rank0 [2026-10-27 23:02:26,674] INFO : plotted sig_proba_elemtype8 +rank0 [2026-10-27 23:02:26,809] INFO : plotted energy_elemtype9 +rank0 [2026-10-27 23:02:26,945] INFO : plotted pt_elemtype9 +rank0 [2026-10-27 23:02:27,022] INFO : plotted energy_elemtype9_corr +rank0 [2026-10-27 23:02:27,098] INFO : plotted pt_elemtype9_corr +rank0 [2026-10-27 23:02:27,173] INFO : plotted eta_elemtype9_corr +rank0 [2026-10-27 23:02:27,248] INFO : plotted sphi_elemtype9_corr +rank0 [2026-10-27 23:02:27,325] INFO : plotted cphi_elemtype9_corr +rank0 [2026-10-27 23:02:28,144] INFO : plotted sig_proba_elemtype9 +rank0 [2026-10-27 23:02:28,255] INFO : plotted energy_elemtype10 +rank0 [2026-10-27 23:02:28,364] INFO : plotted pt_elemtype10 +rank0 [2026-10-27 23:02:28,439] INFO : plotted energy_elemtype10_corr +rank0 [2026-10-27 23:02:28,516] INFO : plotted pt_elemtype10_corr +rank0 [2026-10-27 23:02:28,593] INFO : plotted eta_elemtype10_corr +rank0 [2026-10-27 23:02:28,671] INFO : plotted sphi_elemtype10_corr +rank0 [2026-10-27 23:02:28,749] INFO : plotted cphi_elemtype10_corr +rank0 [2026-10-27 23:02:28,860] INFO : plotted sig_proba_elemtype10 +rank0 [2026-10-27 23:02:28,977] INFO : plotted energy_elemtype11 +rank0 [2026-10-27 23:02:29,093] INFO : plotted pt_elemtype11 +rank0 [2026-10-27 23:02:29,169] INFO : plotted energy_elemtype11_corr +rank0 [2026-10-27 23:02:29,245] INFO : plotted pt_elemtype11_corr +rank0 [2026-10-27 23:02:29,322] INFO : plotted eta_elemtype11_corr +rank0 [2026-10-27 23:02:29,399] INFO : plotted sphi_elemtype11_corr +rank0 [2026-10-27 23:02:29,477] INFO : plotted cphi_elemtype11_corr +rank0 [2026-10-27 23:02:29,592] INFO : plotted sig_proba_elemtype11 +rank0 [2026-10-27 23:02:29,670] INFO : plotted pt_target histogram +rank0 [2026-10-27 23:02:29,673] INFO : plotted pt_pred histogram +rank0 [2026-10-27 23:02:29,676] INFO : plotted pt_ratio histogram +rank0 [2026-10-27 23:02:29,679] INFO : plotted eta_target histogram +rank0 [2026-10-27 23:02:29,682] INFO : plotted eta_pred histogram +rank0 [2026-10-27 23:02:29,685] INFO : plotted eta_ratio histogram +rank0 [2026-10-27 23:02:29,686] INFO : plotted sphi_target histogram +rank0 [2026-10-27 23:02:29,689] INFO : plotted sphi_pred histogram +rank0 [2026-10-27 23:02:29,692] INFO : plotted sphi_ratio histogram +rank0 [2026-10-27 23:02:29,694] INFO : plotted cphi_target histogram +rank0 [2026-10-27 23:02:29,697] INFO : plotted cphi_pred histogram +rank0 [2026-10-27 23:02:29,700] INFO : plotted cphi_ratio histogram +rank0 [2026-10-27 23:02:29,703] INFO : plotted energy_target histogram +rank0 [2026-10-27 23:02:29,705] INFO : plotted energy_pred histogram +rank0 [2026-10-27 23:02:29,708] INFO : plotted energy_ratio histogram +rank0 [2026-10-27 23:04:24,030] INFO : VALIDATION | Step=100000/100000 | Train Loss=11.9488 | Valid Loss=12.6143 | Train Loss Unweighted=3.6450 | Valid Loss Unweighted=4.0121 | Group Valid Loss Unweighted: physical_pu: 3.6161 | physical_nopu: 4.4081 | Stale=0 | ETA=0.0m +rank0 [2026-10-27 23:04:24,038] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 23:04:24,359] INFO : test_dataset: cms_pf_ttbar, 510 +rank0 [2026-10-27 23:04:24,364] INFO : Running predictions on cms_pf_ttbar +rank0 [2026-10-27 23:04:49,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_0_0.parquet +rank0 [2026-10-27 23:04:50,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_0_1.parquet +rank0 [2026-10-27 23:04:50,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_0_2.parquet +rank0 [2026-10-27 23:04:51,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_0_3.parquet +rank0 [2026-10-27 23:04:52,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_0_4.parquet +rank0 [2026-10-27 23:04:52,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_0_5.parquet +rank0 [2026-10-27 23:04:53,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_0_6.parquet +rank0 [2026-10-27 23:04:53,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_0_7.parquet +rank0 [2026-10-27 23:05:03,805] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 23:05:03,811] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 23:05:04,180] INFO : test_dataset: cms_pf_qcd, 510 +rank0 [2026-10-27 23:05:04,184] INFO : Running predictions on cms_pf_qcd +rank0 [2026-10-27 23:05:19,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_0_0.parquet +rank0 [2026-10-27 23:05:20,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_0_1.parquet +rank0 [2026-10-27 23:05:20,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_0_2.parquet +rank0 [2026-10-27 23:05:21,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_0_3.parquet +rank0 [2026-10-27 23:05:22,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_0_4.parquet +rank0 [2026-10-27 23:05:22,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_0_5.parquet +rank0 [2026-10-27 23:05:23,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_0_6.parquet +rank0 [2026-10-27 23:05:23,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_0_7.parquet +rank0 [2026-10-27 23:05:33,761] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 23:05:33,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-27 23:05:34,200] INFO : test_dataset: cms_pf_ztt, 510 +rank0 [2026-10-27 23:05:34,204] INFO : Running predictions on cms_pf_ztt +rank0 [2026-10-27 23:05:49,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_0_0.parquet +rank0 [2026-10-27 23:05:50,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_0_1.parquet +rank0 [2026-10-27 23:05:50,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_0_2.parquet +rank0 [2026-10-27 23:05:51,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_0_3.parquet +rank0 [2026-10-27 23:05:51,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_0_4.parquet +rank0 [2026-10-27 23:05:52,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_0_5.parquet +rank0 [2026-10-27 23:05:52,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_0_6.parquet +rank0 [2026-10-27 23:05:53,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_0_7.parquet +rank0 [2026-10-27 23:06:03,472] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-27 23:06:03,594] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar +rank0 [2026-10-27 23:06:08,878] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar +rank0 [2026-10-27 23:06:10,073] INFO : Plotted number of elements +rank0 [2026-10-27 23:06:11,810] INFO : Plotted jets +rank0 [2026-10-27 23:06:13,002] INFO : Plotted jet ratio +rank0 [2026-10-27 23:06:13,057] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd +rank0 [2026-10-27 23:06:18,377] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd +rank0 [2026-10-27 23:06:19,629] INFO : Plotted number of elements +rank0 [2026-10-27 23:06:21,834] INFO : Plotted jets +rank0 [2026-10-27 23:06:22,989] INFO : Plotted jet ratio +rank0 [2026-10-27 23:06:23,028] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt +rank0 [2026-10-27 23:06:28,199] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt +rank0 [2026-10-27 23:06:29,518] INFO : Plotted number of elements +rank0 [2026-10-27 23:06:31,404] INFO : Plotted jets +rank0 [2026-10-27 23:06:32,620] INFO : Plotted jet ratio +rank0 [2026-10-27 23:06:33,211] INFO : Training completed. Total time on device 0: 415.277min diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.1 b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.1 new file mode 100644 index 0000000000000000000000000000000000000000..a9dec694cd2b617a2da80b802c083af8b522234b --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.1 @@ -0,0 +1,897 @@ +rank1 [2026-10-27 16:10:10,377] INFO : Initializing model with seed=12345; process seed=12346 +rank1 [2026-10-27 16:10:10,378] INFO : configured dtype=torch.bfloat16 for autocast +rank1 [2026-10-27 16:10:35,267] INFO : Instantiating model +rank1 [2026-10-27 16:10:35,267] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank1 [2026-10-27 16:10:35,268] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank1 [2026-10-27 16:10:35,304] INFO : Input encoding initialization took 0.04s +rank1 [2026-10-27 16:10:35,304] INFO : nn0 parameters: 2130944 +rank1 [2026-10-27 16:10:35,304] INFO : Initializing shared backbone layers, num_convs=3 +rank1 [2026-10-27 16:10:35,304] INFO : Initializing backbone layer 0 +rank1 [2026-10-27 16:10:35,350] INFO : Initializing backbone layer 1 +rank1 [2026-10-27 16:10:35,380] INFO : Initializing backbone layer 2 +rank1 [2026-10-27 16:10:35,409] INFO : Backbone initialization took 0.11s +rank1 [2026-10-27 16:10:35,410] INFO : backbone parameters: 15765696 +rank1 [2026-10-27 16:10:35,410] INFO : Initializing output DNNs +rank1 [2026-10-27 16:10:35,444] INFO : Output DNNs initialization took 0.03s +rank1 [2026-10-27 16:10:35,444] INFO : backbone_mode=BackboneMode.SHARED +rank1 [2026-10-27 16:10:35,444] INFO : jagged_attention enabled=False +rank1 [2026-10-27 16:10:35,444] INFO : flash_attn_varlen enabled=False +rank1 [2026-10-27 16:10:35,444] INFO : task_queries enabled=False +rank1 [2026-10-27 16:10:35,444] INFO : classification_norm parameters: 0 +rank1 [2026-10-27 16:10:35,444] INFO : regression_norm parameters: 0 +rank1 [2026-10-27 16:10:35,444] INFO : classification_query parameters: 0 +rank1 [2026-10-27 16:10:35,444] INFO : regression_query parameters: 0 +rank1 [2026-10-27 16:10:35,444] INFO : classification_readout parameters: 0 +rank1 [2026-10-27 16:10:35,444] INFO : regression_readout parameters: 0 +rank1 [2026-10-27 16:10:35,444] INFO : output_mode=elementwise +rank1 [2026-10-27 16:10:35,444] INFO : set_decoder parameters: 0 +rank1 [2026-10-27 16:10:35,444] INFO : nn_binary_particle parameters: 264706 +rank1 [2026-10-27 16:10:35,445] INFO : nn_pid parameters: 268297 +rank1 [2026-10-27 16:10:35,445] INFO : nn_pt parameters: 2113544 +rank1 [2026-10-27 16:10:35,445] INFO : nn_eta parameters: 264706 +rank1 [2026-10-27 16:10:35,445] INFO : nn_sin_phi parameters: 264706 +rank1 [2026-10-27 16:10:35,445] INFO : nn_cos_phi parameters: 264706 +rank1 [2026-10-27 16:10:35,445] INFO : nn_energy parameters: 2113544 +rank1 [2026-10-27 16:10:35,446] INFO : Total MLPF parameters: 23450849 +rank1 [2026-10-27 16:10:35,446] INFO : MLPF __init__ done +rank1 [2026-10-27 16:10:35,446] INFO : Instantiated model +rank1 [2026-10-27 16:10:35,446] INFO : Moving model to device rank=1 +rank1 [2026-10-27 16:10:35,477] INFO : Moved model to device rank=1 +rank1 [2026-10-27 16:10:35,479] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank1 [2026-10-27 16:10:35,479] INFO : Configured model for SyncBatchNorm rank=1 +rank1 [2026-10-27 16:10:36,004] INFO : Configured model for DistributedDataParallel rank=1 +rank1 [2026-10-27 16:10:36,123] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +202 module.nn_energy.nn.7.0.bias ... 0 +203 module.nn_energy.nn.7.2.weight ... 0 +204 module.nn_energy.nn.7.2.bias ... 0 +205 module.nn_energy.nn.7.4.weight ... 0 +206 module.nn_energy.nn.7.4.bias ... 0 + +[207 rows x 3 columns] +rank1 [2026-10-27 16:10:36,123] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x HEPTv2Layer( + embedding_dim=512, num_heads=16, width=512, dropout=0.0, block_size=100, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=512, out_features=512, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=512, out_features=512, bias=False) + (w_k): Linear(in_features=512, out_features=512, bias=False) + (w_v): Linear(in_features=512, out_features=512, bias=False) + (w_rpe): Linear(in_features=10, out_features=512, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=2048, bias=True) + (1): LayerNorm((2048,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=2048, out_features=512, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=512, out_features=2048, bias=False) + (up_proj): Linear(in_features=512, out_features=2048, bias=False) + (down_proj): Linear(in_features=2048, out_features=512, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank1 [2026-10-27 16:10:36,124] INFO : Trainable parameters: 23450849 +rank1 [2026-10-27 16:10:36,124] INFO : Non-trainable parameters: 22386 +rank1 [2026-10-27 16:10:36,124] INFO : Total parameters: 23473235 +rank1 [2026-10-27 16:10:36,126] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 262144 0 + module.backbone.0.attn.out_linear.bias 512 0 + module.backbone.0.attn.e2lsh.alpha 0 1632 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 262144 0 + module.backbone.0.w_k.weight 262144 0 + module.backbone.0.w_v.weight 262144 0 + module.backbone.0.w_rpe.weight 0 5120 + module.backbone.0.w_rpe.bias 0 512 +module.backbone.0.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm2.weight 512 0 + module.backbone.0.q_norm.weight 32 0 + module.backbone.0.k_norm.weight 32 0 + module.backbone.0.ff.gate_proj.weight 1048576 0 + module.backbone.0.ff.up_proj.weight 1048576 0 + module.backbone.0.ff.down_proj.weight 1048576 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 262144 0 + module.backbone.1.attn.out_linear.bias 512 0 + module.backbone.1.attn.e2lsh.alpha 0 1632 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 262144 0 + module.backbone.1.w_k.weight 262144 0 + module.backbone.1.w_v.weight 262144 0 + module.backbone.1.w_rpe.weight 0 5120 + module.backbone.1.w_rpe.bias 0 512 +module.backbone.1.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm2.weight 512 0 + module.backbone.1.q_norm.weight 32 0 + module.backbone.1.k_norm.weight 32 0 + module.backbone.1.ff.gate_proj.weight 1048576 0 + module.backbone.1.ff.up_proj.weight 1048576 0 + module.backbone.1.ff.down_proj.weight 1048576 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 262144 0 + module.backbone.2.attn.out_linear.bias 512 0 + module.backbone.2.attn.e2lsh.alpha 0 1632 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 262144 0 + module.backbone.2.w_k.weight 262144 0 + module.backbone.2.w_v.weight 262144 0 + module.backbone.2.w_rpe.weight 0 5120 + module.backbone.2.w_rpe.bias 0 512 +module.backbone.2.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm2.weight 512 0 + module.backbone.2.q_norm.weight 32 0 + module.backbone.2.k_norm.weight 32 0 + module.backbone.2.ff.gate_proj.weight 1048576 0 + module.backbone.2.ff.up_proj.weight 1048576 0 + module.backbone.2.ff.down_proj.weight 1048576 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank1 [2026-10-27 16:10:36,126] INFO : Getting dataloaders +rank1 [2026-10-27 16:10:36,126] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:43,328] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:44,230] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:44,828] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank1 [2026-10-27 16:10:44,828] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:46,069] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:46,994] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:47,783] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank1 [2026-10-27 16:10:47,783] INFO : Creating WeightedMixIterator with 2 data loaders. +rank1 [2026-10-27 16:10:47,783] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:47,937] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:48,106] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:48,259] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank1 [2026-10-27 16:10:48,259] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:48,425] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:48,608] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:10:48,766] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank1 [2026-10-27 16:10:48,766] INFO : Creating InterleavedIterator with 2 data loaders. +rank1 [2026-10-27 16:11:16,251] INFO : Got dataloaders +rank1 [2026-10-27 16:11:16,251] INFO : loader split=train rank=1 len=232058 +rank1 [2026-10-27 16:11:16,251] INFO : loader split=valid rank=1 len=48 +rank1 [2026-10-27 16:11:16,251] INFO : Starting training from step 1 to 100000 on rank 1 of 8 +rank1 [2026-10-27 16:11:16,251] INFO : Creating train_iterator +rank1 [2026-10-27 16:11:16,251] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank1 [2026-10-27 16:11:30,252] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank1 [2026-10-27 16:11:45,644] INFO : Created train_iterator +rank1 [2026-10-27 16:12:32,261] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.129867712855339, 'Classification': 0.16580097352620216, 'Regression_pt': 0.7168090478330851, 'Regression_eta': 0.335200086296536, 'Regression_sin_phi': 0.1274821354681626, 'Regression_cos_phi': 0.1371471309568733, 'Regression_energy': 0.9554258581250906}, weights={'Classification_binary': 1.0, 'Classification': 30.939912796020508, 'Regression_pt': 7.156533241271973, 'Regression_eta': 15.3038969039917, 'Regression_sin_phi': 40.23989486694336, 'Regression_cos_phi': 37.40412139892578, 'Regression_energy': 5.369194984436035} +rank1 [2026-10-27 16:15:18,322] INFO : Step 1000/100000 rank1 | Train Loss: 16.8632 | Train Loss Unweighted: 5.4934 | Train Loss Components: Classification: 0.0189 | Classification_binary: 4.1833 | Regression_cos_phi: 0.0509 | Regression_energy: 0.5806 | Regression_eta: 0.0776 | Regression_pt: 0.5302 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 5.1301 | physical_nopu: 6.0772 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5866 | Classification_binary: 4.1586 | Regression_cos_phi: 1.7215 | Regression_energy: 3.1256 | Regression_eta: 1.2016 | Regression_pt: 3.7675 | Regression_sin_phi: 2.3208 | LR: 4.00e-04 | DataLoad Time: 0.0010s | Model Forward Time: 2.3151s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0014 | max_forward: 0.0387 | max_loss: 0.0036 | max_backward: 0.0664 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 1.2817 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 4374.6 | uss_MB: 3102.3 | child_rss_MB: 11636.1 | allocated_MB: 609.1 | reserved_MB: 27788.0 | inactive_split_MB: 244.1 +rank1 [2026-10-27 16:18:42,007] INFO : Step 2000/100000 rank1 | Train Loss: 17.7204 | Train Loss Unweighted: 5.4242 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.9240 | Regression_cos_phi: 0.0505 | Regression_energy: 0.6758 | Regression_eta: 0.0665 | Regression_pt: 0.6366 | Regression_sin_phi: 0.0551 | Group Train Loss Unweighted (window mean): physical_pu: 4.2802 | physical_nopu: 5.3470 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5251 | Classification_binary: 3.9394 | Regression_cos_phi: 1.7584 | Regression_energy: 4.1015 | Regression_eta: 1.0166 | Regression_pt: 5.2380 | Regression_sin_phi: 1.6992 | LR: 4.00e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.8062s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1422 | max_loss: 0.0036 | max_backward: 0.1717 | max_optimizer_step: 0.1501 | max_tensorboard_logging: 0.7287 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 4367.5 | uss_MB: 3102.8 | child_rss_MB: 11637.6 | allocated_MB: 607.0 | reserved_MB: 27788.0 | inactive_split_MB: 326.9 +rank1 [2026-10-27 16:22:10,507] INFO : Step 3000/100000 rank1 | Train Loss: 14.3102 | Train Loss Unweighted: 4.8875 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.8970 | Regression_cos_phi: 0.0491 | Regression_energy: 0.4277 | Regression_eta: 0.0716 | Regression_pt: 0.3769 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 4.0705 | physical_nopu: 5.1319 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4371 | Classification_binary: 3.7291 | Regression_cos_phi: 1.3700 | Regression_energy: 2.5039 | Regression_eta: 0.8507 | Regression_pt: 3.1722 | Regression_sin_phi: 1.7478 | LR: 3.99e-04 | DataLoad Time: 0.0013s | Model Forward Time: 2.0076s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0782 | max_loss: 0.0037 | max_backward: 0.0505 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 1.0271 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4295.2 | uss_MB: 3034.9 | child_rss_MB: 11625.5 | allocated_MB: 607.6 | reserved_MB: 28576.0 | inactive_split_MB: 261.0 +rank1 [2026-10-27 16:25:44,194] INFO : Step 4000/100000 rank1 | Train Loss: 12.6160 | Train Loss Unweighted: 3.9080 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.1145 | Regression_cos_phi: 0.0484 | Regression_energy: 0.3088 | Regression_eta: 0.1117 | Regression_pt: 0.2597 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.9908 | physical_nopu: 5.0219 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4739 | Classification_binary: 3.1096 | Regression_cos_phi: 1.8409 | Regression_energy: 1.7731 | Regression_eta: 1.6700 | Regression_pt: 1.9995 | Regression_sin_phi: 2.0267 | LR: 3.99e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.4269s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0032 | max_forward: 0.1365 | max_loss: 0.0036 | max_backward: 0.2496 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 1.0566 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 4292.4 | uss_MB: 3043.1 | child_rss_MB: 11491.5 | allocated_MB: 617.3 | reserved_MB: 28576.0 | inactive_split_MB: 1049.1 +rank1 [2026-10-27 16:29:29,026] INFO : Step 5000/100000 rank1 | Train Loss: 13.7309 | Train Loss Unweighted: 4.7575 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.8762 | Regression_cos_phi: 0.0508 | Regression_energy: 0.3661 | Regression_eta: 0.0922 | Regression_pt: 0.3105 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.9453 | physical_nopu: 4.9561 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4132 | Classification_binary: 3.7736 | Regression_cos_phi: 1.6676 | Regression_energy: 1.8835 | Regression_eta: 1.2439 | Regression_pt: 2.1615 | Regression_sin_phi: 2.4323 | LR: 3.98e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.0873s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.0008 | max_forward: 0.0371 | max_loss: 0.0036 | max_backward: 0.0568 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 2.1174 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4241.6 | uss_MB: 3036.2 | child_rss_MB: 11146.9 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 248.7 +rank1 [2026-10-27 16:33:03,739] INFO : Step 6000/100000 rank1 | Train Loss: 15.2316 | Train Loss Unweighted: 4.9080 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.7479 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5147 | Regression_eta: 0.0679 | Regression_pt: 0.4609 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.9059 | physical_nopu: 4.9043 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4739 | Classification_binary: 3.9034 | Regression_cos_phi: 2.2207 | Regression_energy: 2.1886 | Regression_eta: 0.9627 | Regression_pt: 2.5776 | Regression_sin_phi: 1.7948 | LR: 3.97e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.0045s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0421 | max_loss: 0.0036 | max_backward: 0.0620 | max_optimizer_step: 0.1489 | max_tensorboard_logging: 2.0592 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4216.9 | uss_MB: 3036.3 | child_rss_MB: 11044.6 | allocated_MB: 610.3 | reserved_MB: 28576.0 | inactive_split_MB: 254.0 +rank1 [2026-10-27 16:36:49,903] INFO : Step 7000/100000 rank1 | Train Loss: 14.7215 | Train Loss Unweighted: 4.8917 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8534 | Regression_cos_phi: 0.0503 | Regression_energy: 0.4508 | Regression_eta: 0.0791 | Regression_pt: 0.3921 | Regression_sin_phi: 0.0547 | Group Train Loss Unweighted (window mean): physical_pu: 3.8763 | physical_nopu: 4.8627 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4643 | Classification_binary: 3.7750 | Regression_cos_phi: 1.7774 | Regression_energy: 2.6576 | Regression_eta: 1.3054 | Regression_pt: 3.0485 | Regression_sin_phi: 2.1945 | LR: 3.96e-04 | DataLoad Time: 0.0011s | Model Forward Time: 4.5453s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0348 | max_loss: 0.0036 | max_backward: 0.0535 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 3.5409 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4194.7 | uss_MB: 3033.5 | child_rss_MB: 10983.9 | allocated_MB: 609.0 | reserved_MB: 28576.0 | inactive_split_MB: 216.2 +rank1 [2026-10-27 16:40:34,139] INFO : Step 8000/100000 rank1 | Train Loss: 14.2560 | Train Loss Unweighted: 4.8042 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.8045 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4333 | Regression_eta: 0.0601 | Regression_pt: 0.3883 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.8518 | physical_nopu: 4.8294 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6162 | Classification_binary: 3.7893 | Regression_cos_phi: 1.9286 | Regression_energy: 2.6979 | Regression_eta: 1.0006 | Regression_pt: 2.9687 | Regression_sin_phi: 1.8169 | LR: 3.94e-04 | DataLoad Time: 0.0017s | Model Forward Time: 4.8862s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1342 | max_loss: 0.0036 | max_backward: 0.0500 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 3.8645 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4176.9 | uss_MB: 3036.4 | child_rss_MB: 10934.3 | allocated_MB: 606.6 | reserved_MB: 28576.0 | inactive_split_MB: 255.4 +rank1 [2026-10-27 16:44:13,307] INFO : Step 9000/100000 rank1 | Train Loss: 15.9274 | Train Loss Unweighted: 4.9361 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.6437 | Regression_cos_phi: 0.0472 | Regression_energy: 0.5701 | Regression_eta: 0.0733 | Regression_pt: 0.5360 | Regression_sin_phi: 0.0496 | Group Train Loss Unweighted (window mean): physical_pu: 3.8376 | physical_nopu: 4.8057 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5055 | Classification_binary: 3.5972 | Regression_cos_phi: 2.1404 | Regression_energy: 3.2393 | Regression_eta: 1.1720 | Regression_pt: 4.2719 | Regression_sin_phi: 1.4947 | LR: 3.93e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.2105s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1320 | max_loss: 0.0037 | max_backward: 0.0555 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 2.0797 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 4157.5 | uss_MB: 3035.9 | child_rss_MB: 10877.2 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 321.4 +rank1 [2026-10-27 16:48:05,432] INFO : Step 10000/100000 rank1 | Train Loss: 13.0514 | Train Loss Unweighted: 4.5682 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.7678 | Regression_cos_phi: 0.0520 | Regression_energy: 0.3372 | Regression_eta: 0.0736 | Regression_pt: 0.2745 | Regression_sin_phi: 0.0521 | Group Train Loss Unweighted (window mean): physical_pu: 3.8237 | physical_nopu: 4.7797 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4430 | Classification_binary: 3.7949 | Regression_cos_phi: 1.8981 | Regression_energy: 1.9747 | Regression_eta: 1.2788 | Regression_pt: 2.0332 | Regression_sin_phi: 2.2579 | LR: 3.91e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.2566s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.0379 | max_loss: 0.0040 | max_backward: 0.0556 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 2.2516 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4149.4 | uss_MB: 3036.1 | child_rss_MB: 10872.4 | allocated_MB: 607.9 | reserved_MB: 28576.0 | inactive_split_MB: 258.7 +rank1 [2026-10-27 16:48:06,334] INFO : Running validation on rank1 +rank1 [2026-10-27 16:48:06,334] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 16:48:06,334] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 16:48:06,334] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 16:48:06,334] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 16:50:06,283] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:50:06,771] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 16:50:34,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 16:50:35,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 16:50:36,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 16:50:36,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 16:50:37,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 16:50:37,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 16:50:38,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 16:50:38,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 16:50:48,731] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-27 16:50:48,968] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:50:49,336] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 16:51:05,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 16:51:05,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 16:51:06,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 16:51:06,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 16:51:07,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 16:51:07,515] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 16:51:08,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 16:51:08,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 16:51:18,577] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 16:51:18,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 16:51:19,153] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 16:51:35,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 16:51:35,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 16:51:36,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 16:51:36,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 16:51:37,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 16:51:37,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 16:51:38,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 16:51:38,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 16:51:48,540] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 16:55:58,693] INFO : Step 11000/100000 rank1 | Train Loss: 14.4095 | Train Loss Unweighted: 4.7539 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.7591 | Regression_cos_phi: 0.0522 | Regression_energy: 0.4228 | Regression_eta: 0.0709 | Regression_pt: 0.3796 | Regression_sin_phi: 0.0518 | Group Train Loss Unweighted (window mean): physical_pu: 3.8077 | physical_nopu: 4.7444 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4770 | Classification_binary: 3.7521 | Regression_cos_phi: 2.6922 | Regression_energy: 2.2847 | Regression_eta: 1.1041 | Regression_pt: 2.5302 | Regression_sin_phi: 2.4738 | LR: 3.89e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.1999s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1127 | max_loss: 0.0074 | max_backward: 0.1410 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 1.9404 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6580.9 | uss_MB: 5462.7 | child_rss_MB: 10900.1 | allocated_MB: 610.6 | reserved_MB: 27422.0 | inactive_split_MB: 253.4 +rank1 [2026-10-27 16:59:46,314] INFO : Step 12000/100000 rank1 | Train Loss: 16.4725 | Train Loss Unweighted: 5.0863 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.5958 | Regression_cos_phi: 0.0410 | Regression_energy: 0.6842 | Regression_eta: 0.0610 | Regression_pt: 0.6484 | Regression_sin_phi: 0.0394 | Group Train Loss Unweighted (window mean): physical_pu: 3.8013 | physical_nopu: 4.7308 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4159 | Classification_binary: 3.7179 | Regression_cos_phi: 1.6066 | Regression_energy: 3.7602 | Regression_eta: 1.0365 | Regression_pt: 4.6708 | Regression_sin_phi: 2.0226 | LR: 3.87e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.5371s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1292 | max_loss: 0.0037 | max_backward: 0.1951 | max_optimizer_step: 0.2940 | max_tensorboard_logging: 2.1160 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6552.9 | uss_MB: 5462.9 | child_rss_MB: 10800.4 | allocated_MB: 609.5 | reserved_MB: 27422.0 | inactive_split_MB: 347.2 +rank1 [2026-10-27 17:03:40,557] INFO : Step 13000/100000 rank1 | Train Loss: 12.9993 | Train Loss Unweighted: 4.5752 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.7655 | Regression_cos_phi: 0.0483 | Regression_energy: 0.3375 | Regression_eta: 0.0674 | Regression_pt: 0.2918 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7898 | physical_nopu: 4.7063 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3437 | Classification_binary: 3.7342 | Regression_cos_phi: 2.0189 | Regression_energy: 2.4624 | Regression_eta: 1.5967 | Regression_pt: 2.8518 | Regression_sin_phi: 2.9972 | LR: 3.85e-04 | DataLoad Time: 0.0086s | Model Forward Time: 3.0972s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1327 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.9573 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5436.1 | uss_MB: 4383.7 | child_rss_MB: 10499.2 | allocated_MB: 607.2 | reserved_MB: 28584.0 | inactive_split_MB: 340.8 +rank1 [2026-10-27 17:07:23,686] INFO : Step 14000/100000 rank1 | Train Loss: 12.1035 | Train Loss Unweighted: 3.7686 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.0394 | Regression_cos_phi: 0.0492 | Regression_energy: 0.2767 | Regression_eta: 0.1064 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0483 | Group Train Loss Unweighted (window mean): physical_pu: 3.7803 | physical_nopu: 4.6878 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5309 | Classification_binary: 3.0162 | Regression_cos_phi: 1.8461 | Regression_energy: 1.5368 | Regression_eta: 1.8600 | Regression_pt: 1.6832 | Regression_sin_phi: 1.9596 | LR: 3.83e-04 | DataLoad Time: 0.0017s | Model Forward Time: 5.5488s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0034 | max_forward: 0.2002 | max_loss: 0.0036 | max_backward: 0.3668 | max_optimizer_step: 0.2700 | max_tensorboard_logging: 3.9098 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0824 | Memory: rss_MB: 5443.1 | uss_MB: 4393.2 | child_rss_MB: 10497.0 | allocated_MB: 618.5 | reserved_MB: 28584.0 | inactive_split_MB: 893.5 +rank1 [2026-10-27 17:11:07,917] INFO : Step 15000/100000 rank1 | Train Loss: 13.2483 | Train Loss Unweighted: 4.5172 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.6965 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3441 | Regression_eta: 0.0751 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7702 | physical_nopu: 4.6706 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4912 | Classification_binary: 3.4617 | Regression_cos_phi: 1.5926 | Regression_energy: 2.4988 | Regression_eta: 1.0347 | Regression_pt: 2.5126 | Regression_sin_phi: 1.8795 | LR: 3.80e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.3447s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0879 | max_loss: 0.0036 | max_backward: 0.0859 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 2.2540 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0190 | Memory: rss_MB: 5437.1 | uss_MB: 4385.0 | child_rss_MB: 10498.2 | allocated_MB: 605.9 | reserved_MB: 28584.0 | inactive_split_MB: 314.1 +rank1 [2026-10-27 17:14:55,829] INFO : Step 16000/100000 rank1 | Train Loss: 14.0060 | Train Loss Unweighted: 4.5724 | Train Loss Components: Classification: 0.0150 | Classification_binary: 3.5827 | Regression_cos_phi: 0.0469 | Regression_energy: 0.4276 | Regression_eta: 0.0760 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0517 | Group Train Loss Unweighted (window mean): physical_pu: 3.7654 | physical_nopu: 4.6500 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4531 | Classification_binary: 3.7320 | Regression_cos_phi: 1.8250 | Regression_energy: 2.2684 | Regression_eta: 1.8274 | Regression_pt: 2.5335 | Regression_sin_phi: 1.9321 | LR: 3.78e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.4622s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0984 | max_forward: 0.1749 | max_loss: 0.0039 | max_backward: 0.0976 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 2.1733 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5435.1 | uss_MB: 4384.3 | child_rss_MB: 10495.1 | allocated_MB: 607.6 | reserved_MB: 28978.0 | inactive_split_MB: 330.4 +rank1 [2026-10-27 17:18:33,642] INFO : Step 17000/100000 rank1 | Train Loss: 14.2557 | Train Loss Unweighted: 4.6663 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6292 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4540 | Regression_eta: 0.0715 | Regression_pt: 0.3998 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.7576 | physical_nopu: 4.6430 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4542 | Classification_binary: 3.5467 | Regression_cos_phi: 1.9471 | Regression_energy: 1.9482 | Regression_eta: 1.1736 | Regression_pt: 2.1632 | Regression_sin_phi: 2.4487 | LR: 3.75e-04 | DataLoad Time: 0.0033s | Model Forward Time: 3.9698s | Phase Times: max_pre_step_barrier: 0.5705 | max_batch_to_device: 0.0043 | max_forward: 0.0887 | max_loss: 0.0037 | max_backward: 0.1154 | max_optimizer_step: 0.2542 | max_tensorboard_logging: 2.1801 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5421.0 | uss_MB: 4384.2 | child_rss_MB: 10406.9 | allocated_MB: 609.7 | reserved_MB: 29500.0 | inactive_split_MB: 302.1 +rank1 [2026-10-27 17:22:18,273] INFO : Step 18000/100000 rank1 | Train Loss: 13.2896 | Train Loss Unweighted: 4.5062 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6504 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3607 | Regression_eta: 0.0732 | Regression_pt: 0.3062 | Regression_sin_phi: 0.0530 | Group Train Loss Unweighted (window mean): physical_pu: 3.7500 | physical_nopu: 4.6286 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3541 | Classification_binary: 3.5903 | Regression_cos_phi: 1.6169 | Regression_energy: 1.6875 | Regression_eta: 0.9265 | Regression_pt: 2.0815 | Regression_sin_phi: 1.8063 | LR: 3.72e-04 | DataLoad Time: 0.0012s | Model Forward Time: 10.0059s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0832 | max_loss: 0.0037 | max_backward: 0.0999 | max_optimizer_step: 0.1725 | max_tensorboard_logging: 8.8254 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 5416.7 | uss_MB: 4384.0 | child_rss_MB: 10363.7 | allocated_MB: 608.8 | reserved_MB: 29500.0 | inactive_split_MB: 196.1 +rank1 [2026-10-27 17:25:51,406] INFO : Step 19000/100000 rank1 | Train Loss: 15.2449 | Train Loss Unweighted: 4.8480 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5669 | Regression_cos_phi: 0.0399 | Regression_energy: 0.5792 | Regression_eta: 0.0602 | Regression_pt: 0.5415 | Regression_sin_phi: 0.0443 | Group Train Loss Unweighted (window mean): physical_pu: 3.7399 | physical_nopu: 4.6217 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5935 | Classification_binary: 3.4910 | Regression_cos_phi: 1.6552 | Regression_energy: 5.3194 | Regression_eta: 0.9545 | Regression_pt: 6.9351 | Regression_sin_phi: 1.7990 | LR: 3.69e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.1646s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0008 | max_forward: 0.0818 | max_loss: 0.0762 | max_backward: 0.1759 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 1.9822 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5410.7 | uss_MB: 4384.1 | child_rss_MB: 10330.8 | allocated_MB: 607.1 | reserved_MB: 29500.0 | inactive_split_MB: 236.2 +rank1 [2026-10-27 17:29:45,026] INFO : Step 20000/100000 rank1 | Train Loss: 13.7275 | Train Loss Unweighted: 4.6495 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.7120 | Regression_cos_phi: 0.0510 | Regression_energy: 0.4061 | Regression_eta: 0.0646 | Regression_pt: 0.3531 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.7387 | physical_nopu: 4.6038 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3670 | Classification_binary: 3.7636 | Regression_cos_phi: 1.8405 | Regression_energy: 2.2595 | Regression_eta: 1.0658 | Regression_pt: 2.5713 | Regression_sin_phi: 1.8081 | LR: 3.66e-04 | DataLoad Time: 0.0006s | Model Forward Time: 5.4953s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0007 | max_forward: 0.0851 | max_loss: 0.0039 | max_backward: 0.2016 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 4.2823 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5411.1 | uss_MB: 4383.8 | child_rss_MB: 10307.1 | allocated_MB: 609.5 | reserved_MB: 29500.0 | inactive_split_MB: 252.8 +rank1 [2026-10-27 17:29:45,969] INFO : Running validation on rank1 +rank1 [2026-10-27 17:29:45,969] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 17:29:45,969] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 17:29:45,969] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 17:31:42,604] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 17:31:43,254] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 17:32:09,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 17:32:10,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 17:32:10,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 17:32:11,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 17:32:11,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 17:32:12,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 17:32:12,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 17:32:13,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 17:32:23,199] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 17:32:23,636] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 17:32:24,023] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 17:32:39,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 17:32:39,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 17:32:40,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 17:32:40,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 17:32:41,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 17:32:41,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 17:32:42,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 17:32:42,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 17:32:52,971] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-27 17:32:53,432] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 17:32:53,822] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 17:33:08,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 17:33:09,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 17:33:09,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 17:33:10,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 17:33:10,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 17:33:11,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 17:33:11,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 17:33:12,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 17:33:22,221] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 17:37:33,418] INFO : Step 21000/100000 rank1 | Train Loss: 13.1647 | Train Loss Unweighted: 4.3967 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.5106 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3742 | Regression_eta: 0.0796 | Regression_pt: 0.3229 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.7328 | physical_nopu: 4.5933 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5484 | Classification_binary: 3.6581 | Regression_cos_phi: 1.3155 | Regression_energy: 1.9348 | Regression_eta: 1.3867 | Regression_pt: 2.1790 | Regression_sin_phi: 1.7833 | LR: 3.62e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.6671s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0483 | max_forward: 0.1678 | max_loss: 0.0037 | max_backward: 0.1430 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 2.3569 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5516.5 | uss_MB: 4216.2 | child_rss_MB: 10241.6 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 238.6 +rank1 [2026-10-27 17:41:10,019] INFO : Step 22000/100000 rank1 | Train Loss: 15.3802 | Train Loss Unweighted: 4.7914 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.5311 | Regression_cos_phi: 0.0475 | Regression_energy: 0.5593 | Regression_eta: 0.0730 | Regression_pt: 0.5219 | Regression_sin_phi: 0.0434 | Group Train Loss Unweighted (window mean): physical_pu: 3.7290 | physical_nopu: 4.5858 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4799 | Classification_binary: 3.5602 | Regression_cos_phi: 2.0891 | Regression_energy: 3.6786 | Regression_eta: 1.7831 | Regression_pt: 4.6578 | Regression_sin_phi: 1.6545 | LR: 3.59e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.1739s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0905 | max_loss: 0.0038 | max_backward: 0.1361 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 2.0169 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5517.5 | uss_MB: 4498.8 | child_rss_MB: 10247.2 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 321.0 +rank1 [2026-10-27 17:44:47,651] INFO : Step 23000/100000 rank1 | Train Loss: 12.9848 | Train Loss Unweighted: 4.5226 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.7377 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3326 | Regression_eta: 0.0739 | Regression_pt: 0.2617 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.7288 | physical_nopu: 4.5774 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2791 | Classification_binary: 3.7525 | Regression_cos_phi: 1.9551 | Regression_energy: 1.6923 | Regression_eta: 1.3561 | Regression_pt: 1.7470 | Regression_sin_phi: 2.4907 | LR: 3.55e-04 | DataLoad Time: 0.0069s | Model Forward Time: 3.1589s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0994 | max_loss: 0.0039 | max_backward: 0.1244 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.9838 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 5517.5 | uss_MB: 4498.0 | child_rss_MB: 10251.2 | allocated_MB: 608.2 | reserved_MB: 29422.0 | inactive_split_MB: 218.5 +rank1 [2026-10-27 17:48:44,022] INFO : Step 24000/100000 rank1 | Train Loss: 16.3896 | Train Loss Unweighted: 4.8949 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4510 | Regression_cos_phi: 0.0465 | Regression_energy: 0.6613 | Regression_eta: 0.0617 | Regression_pt: 0.6132 | Regression_sin_phi: 0.0453 | Group Train Loss Unweighted (window mean): physical_pu: 3.7170 | physical_nopu: 4.5767 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5672 | Classification_binary: 3.3975 | Regression_cos_phi: 1.6154 | Regression_energy: 3.7609 | Regression_eta: 0.8315 | Regression_pt: 4.6598 | Regression_sin_phi: 1.6799 | LR: 3.51e-04 | DataLoad Time: 0.0026s | Model Forward Time: 3.1024s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0398 | max_forward: 0.1314 | max_loss: 0.0037 | max_backward: 0.0994 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 1.8290 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5518.7 | uss_MB: 4498.4 | child_rss_MB: 10238.8 | allocated_MB: 606.3 | reserved_MB: 29422.0 | inactive_split_MB: 288.6 +rank1 [2026-10-27 17:52:29,098] INFO : Step 25000/100000 rank1 | Train Loss: 13.0062 | Train Loss Unweighted: 4.4048 | Train Loss Components: Classification: 0.0114 | Classification_binary: 3.6384 | Regression_cos_phi: 0.0552 | Regression_energy: 0.3153 | Regression_eta: 0.0680 | Regression_pt: 0.2574 | Regression_sin_phi: 0.0590 | Group Train Loss Unweighted (window mean): physical_pu: 3.7138 | physical_nopu: 4.5644 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4106 | Classification_binary: 3.6322 | Regression_cos_phi: 2.4384 | Regression_energy: 2.1079 | Regression_eta: 1.0921 | Regression_pt: 2.3956 | Regression_sin_phi: 3.1042 | LR: 3.47e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.5990s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0007 | max_forward: 0.1394 | max_loss: 0.0036 | max_backward: 0.2004 | max_optimizer_step: 0.1603 | max_tensorboard_logging: 1.3791 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5517.7 | uss_MB: 4498.3 | child_rss_MB: 10255.6 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 213.9 +rank1 [2026-10-27 17:56:10,619] INFO : Step 26000/100000 rank1 | Train Loss: 13.5469 | Train Loss Unweighted: 4.4523 | Train Loss Components: Classification: 0.0185 | Classification_binary: 3.5246 | Regression_cos_phi: 0.0498 | Regression_energy: 0.3935 | Regression_eta: 0.0732 | Regression_pt: 0.3461 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5624 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5349 | Classification_binary: 3.6389 | Regression_cos_phi: 1.5740 | Regression_energy: 2.1858 | Regression_eta: 1.0946 | Regression_pt: 2.5745 | Regression_sin_phi: 2.0188 | LR: 3.43e-04 | DataLoad Time: 0.0015s | Model Forward Time: 3.1216s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0838 | max_forward: 0.1679 | max_loss: 0.0039 | max_backward: 0.0991 | max_optimizer_step: 0.2674 | max_tensorboard_logging: 1.6325 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5515.7 | uss_MB: 4498.4 | child_rss_MB: 10248.9 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 201.9 +rank1 [2026-10-27 17:59:48,627] INFO : Step 27000/100000 rank1 | Train Loss: 17.1780 | Train Loss Unweighted: 5.0531 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.5146 | Regression_cos_phi: 0.0448 | Regression_energy: 0.6955 | Regression_eta: 0.0778 | Regression_pt: 0.6583 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5579 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4709 | Classification_binary: 3.7058 | Regression_cos_phi: 2.0609 | Regression_energy: 2.7469 | Regression_eta: 1.1815 | Regression_pt: 3.2678 | Regression_sin_phi: 2.1351 | LR: 3.39e-04 | DataLoad Time: 0.0011s | Model Forward Time: 2.4485s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1019 | max_loss: 0.0489 | max_backward: 0.1596 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5507.7 | uss_MB: 4499.5 | child_rss_MB: 10244.8 | allocated_MB: 611.7 | reserved_MB: 30248.0 | inactive_split_MB: 336.3 +rank1 [2026-10-27 18:03:26,754] INFO : Step 28000/100000 rank1 | Train Loss: 12.6721 | Train Loss Unweighted: 4.3436 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5550 | Regression_cos_phi: 0.0488 | Regression_energy: 0.3264 | Regression_eta: 0.0754 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7057 | physical_nopu: 4.5482 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3604 | Classification_binary: 3.7024 | Regression_cos_phi: 2.0657 | Regression_energy: 1.7948 | Regression_eta: 1.1092 | Regression_pt: 2.0195 | Regression_sin_phi: 2.4401 | LR: 3.35e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.0161s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0898 | max_loss: 0.0057 | max_backward: 0.1164 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.8021 | max_tensorboard_flush: 0.0040 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 5489.8 | uss_MB: 4496.9 | child_rss_MB: 10097.2 | allocated_MB: 606.6 | reserved_MB: 30248.0 | inactive_split_MB: 278.7 +rank1 [2026-10-27 18:07:25,425] INFO : Step 29000/100000 rank1 | Train Loss: 14.8403 | Train Loss Unweighted: 4.5753 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.4458 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4901 | Regression_eta: 0.0767 | Regression_pt: 0.4459 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.7068 | physical_nopu: 4.5439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4902 | Classification_binary: 3.3886 | Regression_cos_phi: 1.6634 | Regression_energy: 2.7866 | Regression_eta: 1.1932 | Regression_pt: 3.3510 | Regression_sin_phi: 1.9652 | LR: 3.30e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.3382s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1464 | max_loss: 0.0037 | max_backward: 0.0935 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 1.1535 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5489.8 | uss_MB: 4498.4 | child_rss_MB: 10094.9 | allocated_MB: 607.4 | reserved_MB: 30248.0 | inactive_split_MB: 313.3 +rank1 [2026-10-27 18:11:12,494] INFO : Step 30000/100000 rank1 | Train Loss: 13.2916 | Train Loss Unweighted: 4.4954 | Train Loss Components: Classification: 0.0126 | Classification_binary: 3.5989 | Regression_cos_phi: 0.0482 | Regression_energy: 0.3865 | Regression_eta: 0.0707 | Regression_pt: 0.3291 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7005 | physical_nopu: 4.5378 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3502 | Classification_binary: 3.7044 | Regression_cos_phi: 1.8078 | Regression_energy: 2.0158 | Regression_eta: 1.2656 | Regression_pt: 2.2338 | Regression_sin_phi: 1.8144 | LR: 3.26e-04 | DataLoad Time: 0.0010s | Model Forward Time: 2.7239s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0805 | max_loss: 0.0038 | max_backward: 0.0926 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 1.5279 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 5479.8 | uss_MB: 4498.3 | child_rss_MB: 10016.6 | allocated_MB: 608.2 | reserved_MB: 30760.0 | inactive_split_MB: 232.9 +rank1 [2026-10-27 18:11:13,431] INFO : Running validation on rank1 +rank1 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 18:13:07,586] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 18:13:08,046] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 18:13:34,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 18:13:34,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 18:13:35,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 18:13:36,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 18:13:36,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 18:13:37,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 18:13:37,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 18:13:38,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 18:13:48,053] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 18:13:48,509] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 18:13:49,356] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 18:14:05,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 18:14:05,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 18:14:06,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 18:14:06,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 18:14:07,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 18:14:07,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 18:14:08,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 18:14:08,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 18:14:18,726] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-27 18:14:18,746] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 18:14:19,219] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 18:14:33,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 18:14:34,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 18:14:34,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 18:14:35,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 18:14:36,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 18:14:36,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 18:14:37,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 18:14:37,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 18:14:47,555] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 18:19:00,303] INFO : Step 31000/100000 rank1 | Train Loss: 12.7750 | Train Loss Unweighted: 4.3889 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5406 | Regression_cos_phi: 0.0465 | Regression_energy: 0.3622 | Regression_eta: 0.0700 | Regression_pt: 0.3101 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6964 | physical_nopu: 4.5360 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4367 | Classification_binary: 3.4774 | Regression_cos_phi: 1.7706 | Regression_energy: 2.4607 | Regression_eta: 1.5559 | Regression_pt: 2.4853 | Regression_sin_phi: 1.8015 | LR: 3.21e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.3239s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.2346 | max_loss: 0.0037 | max_backward: 0.1255 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 1.8384 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1395 | Memory: rss_MB: 6773.5 | uss_MB: 5791.8 | child_rss_MB: 10011.8 | allocated_MB: 609.3 | reserved_MB: 29142.0 | inactive_split_MB: 278.5 +rank1 [2026-10-27 18:22:44,787] INFO : Step 32000/100000 rank1 | Train Loss: 14.3617 | Train Loss Unweighted: 4.6225 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4934 | Regression_cos_phi: 0.0448 | Regression_energy: 0.5043 | Regression_eta: 0.0718 | Regression_pt: 0.4507 | Regression_sin_phi: 0.0414 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5314 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5081 | Classification_binary: 3.3231 | Regression_cos_phi: 1.5080 | Regression_energy: 2.7327 | Regression_eta: 1.0817 | Regression_pt: 3.2274 | Regression_sin_phi: 1.4151 | LR: 3.16e-04 | DataLoad Time: 0.0103s | Model Forward Time: 2.5623s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1373 | max_loss: 0.0041 | max_backward: 0.2040 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 1.2065 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5556.1 | uss_MB: 4574.8 | child_rss_MB: 10008.9 | allocated_MB: 606.1 | reserved_MB: 29142.0 | inactive_split_MB: 511.2 +rank1 [2026-10-27 18:26:33,025] INFO : Step 33000/100000 rank1 | Train Loss: 12.0000 | Train Loss Unweighted: 3.7115 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9674 | Regression_cos_phi: 0.0474 | Regression_energy: 0.2863 | Regression_eta: 0.1056 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4686 | Classification_binary: 2.9909 | Regression_cos_phi: 1.7013 | Regression_energy: 1.5683 | Regression_eta: 1.5841 | Regression_pt: 1.7789 | Regression_sin_phi: 1.8817 | LR: 3.12e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.5986s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0030 | max_forward: 0.1619 | max_loss: 0.0119 | max_backward: 0.2845 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.1927 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5557.1 | uss_MB: 4556.3 | child_rss_MB: 10013.2 | allocated_MB: 623.7 | reserved_MB: 29572.0 | inactive_split_MB: 1441.5 +rank1 [2026-10-27 18:30:15,971] INFO : Step 34000/100000 rank1 | Train Loss: 15.0627 | Train Loss Unweighted: 4.6899 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4670 | Regression_cos_phi: 0.0454 | Regression_energy: 0.5438 | Regression_eta: 0.0626 | Regression_pt: 0.5082 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6889 | physical_nopu: 4.5215 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4983 | Classification_binary: 3.5216 | Regression_cos_phi: 1.8026 | Regression_energy: 3.1115 | Regression_eta: 1.0336 | Regression_pt: 3.6837 | Regression_sin_phi: 2.4819 | LR: 3.07e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.3752s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0007 | max_forward: 0.1346 | max_loss: 0.0037 | max_backward: 0.1938 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1506 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5552.6 | uss_MB: 4573.6 | child_rss_MB: 9984.9 | allocated_MB: 609.6 | reserved_MB: 29572.0 | inactive_split_MB: 230.4 +rank1 [2026-10-27 18:33:55,272] INFO : Step 35000/100000 rank1 | Train Loss: 12.2019 | Train Loss Unweighted: 4.3540 | Train Loss Components: Classification: 0.0089 | Classification_binary: 3.6426 | Regression_cos_phi: 0.0532 | Regression_energy: 0.2965 | Regression_eta: 0.0664 | Regression_pt: 0.2370 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6854 | physical_nopu: 4.5184 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2206 | Classification_binary: 3.6809 | Regression_cos_phi: 2.0175 | Regression_energy: 1.5005 | Regression_eta: 1.4224 | Regression_pt: 1.4801 | Regression_sin_phi: 1.9577 | LR: 3.02e-04 | DataLoad Time: 0.0011s | Model Forward Time: 2.6478s | Phase Times: max_pre_step_barrier: 0.0511 | max_batch_to_device: 0.0857 | max_forward: 0.1832 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 1.2656 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 5548.6 | uss_MB: 4574.4 | child_rss_MB: 9918.7 | allocated_MB: 608.2 | reserved_MB: 29572.0 | inactive_split_MB: 245.8 +rank1 [2026-10-27 18:37:33,850] INFO : Step 36000/100000 rank1 | Train Loss: 12.7130 | Train Loss Unweighted: 4.3877 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.5479 | Regression_cos_phi: 0.0433 | Regression_energy: 0.3521 | Regression_eta: 0.0636 | Regression_pt: 0.3167 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.6860 | physical_nopu: 4.5137 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5291 | Classification_binary: 3.6884 | Regression_cos_phi: 1.3345 | Regression_energy: 1.6520 | Regression_eta: 0.8145 | Regression_pt: 2.0436 | Regression_sin_phi: 1.6884 | LR: 2.97e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.5819s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0007 | max_forward: 0.1685 | max_loss: 0.0137 | max_backward: 0.0851 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.2906 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 5532.6 | uss_MB: 4487.8 | child_rss_MB: 9818.0 | allocated_MB: 609.0 | reserved_MB: 29572.0 | inactive_split_MB: 268.4 +rank1 [2026-10-27 18:41:12,499] INFO : Step 37000/100000 rank1 | Train Loss: 16.1273 | Train Loss Unweighted: 4.7522 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4049 | Regression_cos_phi: 0.0495 | Regression_energy: 0.5975 | Regression_eta: 0.0881 | Regression_pt: 0.5494 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6829 | physical_nopu: 4.5107 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5001 | Classification_binary: 3.4260 | Regression_cos_phi: 1.3286 | Regression_energy: 2.7150 | Regression_eta: 1.2173 | Regression_pt: 3.1289 | Regression_sin_phi: 1.3178 | LR: 2.91e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.3167s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0884 | max_loss: 0.0036 | max_backward: 0.0963 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 1.0635 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0656 | Memory: rss_MB: 5529.2 | uss_MB: 4574.6 | child_rss_MB: 9790.1 | allocated_MB: 605.6 | reserved_MB: 29572.0 | inactive_split_MB: 233.2 +rank1 [2026-10-27 18:44:44,282] INFO : Step 38000/100000 rank1 | Train Loss: 12.5142 | Train Loss Unweighted: 4.4218 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.6290 | Regression_cos_phi: 0.0475 | Regression_energy: 0.3376 | Regression_eta: 0.0660 | Regression_pt: 0.2827 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6875 | physical_nopu: 4.5136 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4021 | Classification_binary: 3.5343 | Regression_cos_phi: 1.6675 | Regression_energy: 1.9232 | Regression_eta: 0.8673 | Regression_pt: 2.0657 | Regression_sin_phi: 1.8689 | LR: 2.86e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.4326s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.0807 | max_loss: 0.0039 | max_backward: 0.1583 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.2386 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5519.2 | uss_MB: 4574.3 | child_rss_MB: 9713.9 | allocated_MB: 607.7 | reserved_MB: 29572.0 | inactive_split_MB: 203.1 +rank1 [2026-10-27 18:48:30,202] INFO : Step 39000/100000 rank1 | Train Loss: 13.9834 | Train Loss Unweighted: 4.4509 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4391 | Regression_cos_phi: 0.0477 | Regression_energy: 0.4363 | Regression_eta: 0.0792 | Regression_pt: 0.3839 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.6802 | physical_nopu: 4.5058 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4796 | Classification_binary: 3.3495 | Regression_cos_phi: 2.0066 | Regression_energy: 1.8235 | Regression_eta: 0.9672 | Regression_pt: 2.1785 | Regression_sin_phi: 1.8920 | LR: 2.81e-04 | DataLoad Time: 0.0026s | Model Forward Time: 3.8719s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.0950 | max_loss: 0.0039 | max_backward: 0.1102 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 2.6970 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5506.6 | uss_MB: 4574.6 | child_rss_MB: 9623.2 | allocated_MB: 611.4 | reserved_MB: 29682.0 | inactive_split_MB: 276.6 +rank1 [2026-10-27 18:52:07,824] INFO : Step 40000/100000 rank1 | Train Loss: 15.4467 | Train Loss Unweighted: 4.8241 | Train Loss Components: Classification: 0.0121 | Classification_binary: 3.5538 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5661 | Regression_eta: 0.0744 | Regression_pt: 0.5256 | Regression_sin_phi: 0.0462 | Group Train Loss Unweighted (window mean): physical_pu: 3.6754 | physical_nopu: 4.5045 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3525 | Classification_binary: 3.7342 | Regression_cos_phi: 1.8114 | Regression_energy: 3.2446 | Regression_eta: 1.1703 | Regression_pt: 3.9626 | Regression_sin_phi: 2.2947 | LR: 2.76e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.4739s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.1251 | max_loss: 0.0037 | max_backward: 0.0913 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2799 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5491.6 | uss_MB: 4574.5 | child_rss_MB: 9510.6 | allocated_MB: 607.6 | reserved_MB: 29682.0 | inactive_split_MB: 251.4 +rank1 [2026-10-27 18:52:08,683] INFO : Running validation on rank1 +rank1 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 18:54:11,846] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 18:54:12,296] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 18:54:37,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 18:54:37,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 18:54:38,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 18:54:38,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 18:54:39,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 18:54:40,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 18:54:40,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 18:54:41,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 18:54:51,039] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-27 18:54:51,317] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 18:54:51,717] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 18:55:07,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 18:55:08,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 18:55:08,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 18:55:09,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 18:55:09,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 18:55:10,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 18:55:10,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 18:55:11,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 18:55:21,159] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 18:55:21,167] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 18:55:21,564] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 18:55:36,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 18:55:37,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 18:55:37,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 18:55:38,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 18:55:38,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 18:55:39,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 18:55:39,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 18:55:40,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 18:55:50,264] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-27 19:00:08,492] INFO : Step 41000/100000 rank1 | Train Loss: 13.2101 | Train Loss Unweighted: 4.4466 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5620 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3777 | Regression_eta: 0.0720 | Regression_pt: 0.3238 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6774 | physical_nopu: 4.4925 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3718 | Classification_binary: 3.5865 | Regression_cos_phi: 1.7030 | Regression_energy: 1.8954 | Regression_eta: 1.7623 | Regression_pt: 2.1076 | Regression_sin_phi: 1.7754 | LR: 2.70e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.5583s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0904 | max_forward: 0.2227 | max_loss: 0.0038 | max_backward: 0.1239 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 2.1580 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6748.4 | uss_MB: 5831.9 | child_rss_MB: 9514.0 | allocated_MB: 608.3 | reserved_MB: 27230.0 | inactive_split_MB: 387.2 +rank1 [2026-10-27 19:03:45,853] INFO : Step 42000/100000 rank1 | Train Loss: 13.7061 | Train Loss Unweighted: 4.4760 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4108 | Regression_cos_phi: 0.0459 | Regression_energy: 0.4702 | Regression_eta: 0.0614 | Regression_pt: 0.4341 | Regression_sin_phi: 0.0374 | Group Train Loss Unweighted (window mean): physical_pu: 3.6711 | physical_nopu: 4.4967 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4831 | Classification_binary: 3.3968 | Regression_cos_phi: 1.5489 | Regression_energy: 2.8898 | Regression_eta: 0.7145 | Regression_pt: 3.7528 | Regression_sin_phi: 1.2264 | LR: 2.65e-04 | DataLoad Time: 0.0073s | Model Forward Time: 2.9855s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0008 | max_forward: 0.1920 | max_loss: 0.0038 | max_backward: 0.1427 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 1.6213 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6739.4 | uss_MB: 5628.8 | child_rss_MB: 9436.4 | allocated_MB: 609.4 | reserved_MB: 27632.0 | inactive_split_MB: 509.5 +rank1 [2026-10-27 19:07:23,507] INFO : Step 43000/100000 rank1 | Train Loss: 11.9489 | Train Loss Unweighted: 3.6938 | Train Loss Components: Classification: 0.0162 | Classification_binary: 2.9682 | Regression_cos_phi: 0.0472 | Regression_energy: 0.2764 | Regression_eta: 0.1089 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6702 | physical_nopu: 4.4944 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5154 | Classification_binary: 2.9054 | Regression_cos_phi: 1.7323 | Regression_energy: 1.4706 | Regression_eta: 1.7684 | Regression_pt: 1.6116 | Regression_sin_phi: 1.9601 | LR: 2.59e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.8334s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0032 | max_forward: 0.2195 | max_loss: 0.0114 | max_backward: 0.4513 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 2.0889 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0865 | Memory: rss_MB: 5560.5 | uss_MB: 3524.0 | child_rss_MB: 9348.2 | allocated_MB: 619.9 | reserved_MB: 27918.0 | inactive_split_MB: 1305.5 +rank1 [2026-10-27 19:11:14,851] INFO : Step 44000/100000 rank1 | Train Loss: 15.8790 | Train Loss Unweighted: 4.7893 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4610 | Regression_cos_phi: 0.0438 | Regression_energy: 0.5944 | Regression_eta: 0.0747 | Regression_pt: 0.5488 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6681 | physical_nopu: 4.4856 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5010 | Classification_binary: 3.7438 | Regression_cos_phi: 1.9089 | Regression_energy: 2.8482 | Regression_eta: 1.0957 | Regression_pt: 3.4500 | Regression_sin_phi: 2.2171 | LR: 2.54e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.1648s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0980 | max_loss: 0.0531 | max_backward: 0.1371 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.9403 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5553.5 | uss_MB: 4666.8 | child_rss_MB: 9287.7 | allocated_MB: 606.4 | reserved_MB: 31154.0 | inactive_split_MB: 295.0 +rank1 [2026-10-27 19:15:01,382] INFO : Step 45000/100000 rank1 | Train Loss: 13.8786 | Train Loss Unweighted: 4.5241 | Train Loss Components: Classification: 0.0100 | Classification_binary: 3.5822 | Regression_cos_phi: 0.0536 | Regression_energy: 0.4068 | Regression_eta: 0.0847 | Regression_pt: 0.3344 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 3.6667 | physical_nopu: 4.4836 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3027 | Classification_binary: 3.6078 | Regression_cos_phi: 1.8737 | Regression_energy: 1.7440 | Regression_eta: 0.9669 | Regression_pt: 2.0178 | Regression_sin_phi: 1.8289 | LR: 2.48e-04 | DataLoad Time: 0.0016s | Model Forward Time: 4.6041s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0009 | max_forward: 0.0951 | max_loss: 0.0815 | max_backward: 0.1837 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 3.3466 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 5551.4 | uss_MB: 4666.0 | child_rss_MB: 9284.1 | allocated_MB: 608.8 | reserved_MB: 31154.0 | inactive_split_MB: 289.8 +rank1 [2026-10-27 19:18:42,715] INFO : Step 46000/100000 rank1 | Train Loss: 13.1587 | Train Loss Unweighted: 4.3779 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.4817 | Regression_cos_phi: 0.0459 | Regression_energy: 0.3805 | Regression_eta: 0.0668 | Regression_pt: 0.3375 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.6620 | physical_nopu: 4.4831 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6180 | Classification_binary: 3.3628 | Regression_cos_phi: 2.2574 | Regression_energy: 2.0757 | Regression_eta: 0.7540 | Regression_pt: 2.6096 | Regression_sin_phi: 1.5915 | LR: 2.43e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.3332s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0924 | max_forward: 0.1910 | max_loss: 0.0039 | max_backward: 0.1028 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 2.0151 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5545.4 | uss_MB: 4666.2 | child_rss_MB: 9248.1 | allocated_MB: 609.2 | reserved_MB: 31154.0 | inactive_split_MB: 277.4 +rank1 [2026-10-27 19:22:31,522] INFO : Step 47000/100000 rank1 | Train Loss: 15.2118 | Train Loss Unweighted: 4.6831 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3624 | Regression_cos_phi: 0.0409 | Regression_energy: 0.5987 | Regression_eta: 0.0659 | Regression_pt: 0.5598 | Regression_sin_phi: 0.0405 | Group Train Loss Unweighted (window mean): physical_pu: 3.6609 | physical_nopu: 4.4795 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4884 | Classification_binary: 3.3792 | Regression_cos_phi: 1.4870 | Regression_energy: 2.8905 | Regression_eta: 0.8797 | Regression_pt: 3.4322 | Regression_sin_phi: 1.3878 | LR: 2.37e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.2741s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0777 | max_forward: 0.1864 | max_loss: 0.0077 | max_backward: 0.1105 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 1.8392 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5533.2 | uss_MB: 4665.8 | child_rss_MB: 9129.7 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 305.0 +rank1 [2026-10-27 19:26:21,057] INFO : Step 48000/100000 rank1 | Train Loss: 12.0455 | Train Loss Unweighted: 4.3014 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.6150 | Regression_cos_phi: 0.0526 | Regression_energy: 0.2805 | Regression_eta: 0.0652 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6617 | physical_nopu: 4.4726 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3656 | Classification_binary: 3.4844 | Regression_cos_phi: 2.0284 | Regression_energy: 1.3313 | Regression_eta: 0.7637 | Regression_pt: 1.4495 | Regression_sin_phi: 1.6939 | LR: 2.31e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.3650s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0531 | max_forward: 0.1607 | max_loss: 0.0044 | max_backward: 0.0997 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 2.0345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5534.2 | uss_MB: 4664.3 | child_rss_MB: 9129.2 | allocated_MB: 606.8 | reserved_MB: 31154.0 | inactive_split_MB: 262.3 +rank1 [2026-10-27 19:30:06,213] INFO : Step 49000/100000 rank1 | Train Loss: 13.3973 | Train Loss Unweighted: 4.4035 | Train Loss Components: Classification: 0.0168 | Classification_binary: 3.5033 | Regression_cos_phi: 0.0506 | Regression_energy: 0.3797 | Regression_eta: 0.0752 | Regression_pt: 0.3299 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6566 | physical_nopu: 4.4735 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4994 | Classification_binary: 3.4185 | Regression_cos_phi: 2.3312 | Regression_energy: 2.1689 | Regression_eta: 0.9509 | Regression_pt: 2.4221 | Regression_sin_phi: 1.9791 | LR: 2.26e-04 | DataLoad Time: 0.0022s | Model Forward Time: 4.5485s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0220 | max_forward: 0.1075 | max_loss: 0.0037 | max_backward: 0.1020 | max_optimizer_step: 0.2722 | max_tensorboard_logging: 3.2606 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5532.9 | uss_MB: 4389.0 | child_rss_MB: 9133.5 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 312.3 +rank1 [2026-10-27 19:33:47,825] INFO : Step 50000/100000 rank1 | Train Loss: 14.2027 | Train Loss Unweighted: 4.5299 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4272 | Regression_cos_phi: 0.0432 | Regression_energy: 0.4877 | Regression_eta: 0.0670 | Regression_pt: 0.4429 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4710 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3396 | Classification_binary: 3.5477 | Regression_cos_phi: 1.9172 | Regression_energy: 1.9384 | Regression_eta: 1.1258 | Regression_pt: 2.1561 | Regression_sin_phi: 1.7796 | LR: 2.20e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.0500s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0964 | max_loss: 0.0590 | max_backward: 0.1520 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.8031 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5519.9 | uss_MB: 4665.8 | child_rss_MB: 9015.1 | allocated_MB: 607.2 | reserved_MB: 31154.0 | inactive_split_MB: 255.6 +rank1 [2026-10-27 19:33:48,728] INFO : Running validation on rank1 +rank1 [2026-10-27 19:33:48,728] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 19:33:48,728] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 19:33:48,728] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 19:35:46,022] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 19:35:46,496] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 19:36:11,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 19:36:12,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 19:36:13,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 19:36:13,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 19:36:14,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 19:36:14,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 19:36:15,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 19:36:15,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 19:36:25,640] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-27 19:36:25,966] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 19:36:26,396] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 19:36:41,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 19:36:42,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 19:36:42,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 19:36:43,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 19:36:43,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 19:36:44,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 19:36:44,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 19:36:45,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 19:36:55,244] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-27 19:36:55,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 19:36:55,958] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 19:37:11,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 19:37:11,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 19:37:12,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 19:37:12,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 19:37:13,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 19:37:13,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 19:37:14,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 19:37:14,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 19:37:24,576] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 19:41:35,792] INFO : Step 51000/100000 rank1 | Train Loss: 13.0997 | Train Loss Unweighted: 4.3733 | Train Loss Components: Classification: 0.0129 | Classification_binary: 3.5282 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3598 | Regression_eta: 0.0785 | Regression_pt: 0.2920 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6545 | physical_nopu: 4.4631 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3279 | Classification_binary: 3.8047 | Regression_cos_phi: 1.9367 | Regression_energy: 1.4971 | Regression_eta: 1.0817 | Regression_pt: 1.5393 | Regression_sin_phi: 1.6706 | LR: 2.14e-04 | DataLoad Time: 0.0106s | Model Forward Time: 3.1048s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1313 | max_loss: 0.0271 | max_backward: 0.1689 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.7590 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6858.3 | uss_MB: 6017.9 | child_rss_MB: 8958.7 | allocated_MB: 607.6 | reserved_MB: 29536.0 | inactive_split_MB: 388.7 +rank1 [2026-10-27 19:45:15,993] INFO : Step 52000/100000 rank1 | Train Loss: 11.7496 | Train Loss Unweighted: 3.6574 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9475 | Regression_cos_phi: 0.0469 | Regression_energy: 0.2717 | Regression_eta: 0.1056 | Regression_pt: 0.2231 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6567 | physical_nopu: 4.4685 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4867 | Classification_binary: 2.8682 | Regression_cos_phi: 1.7687 | Regression_energy: 1.4659 | Regression_eta: 1.5832 | Regression_pt: 1.6143 | Regression_sin_phi: 1.7900 | LR: 2.09e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.6141s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.1059 | max_forward: 0.2956 | max_loss: 0.0038 | max_backward: 0.4187 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 1.8177 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6858.3 | uss_MB: 6026.1 | child_rss_MB: 8942.2 | allocated_MB: 619.9 | reserved_MB: 29990.0 | inactive_split_MB: 1078.6 +rank1 [2026-10-27 19:48:51,528] INFO : Step 53000/100000 rank1 | Train Loss: 12.5294 | Train Loss Unweighted: 4.3083 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5302 | Regression_cos_phi: 0.0494 | Regression_energy: 0.3242 | Regression_eta: 0.0722 | Regression_pt: 0.2705 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6530 | physical_nopu: 4.4629 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3318 | Classification_binary: 3.6571 | Regression_cos_phi: 1.9340 | Regression_energy: 1.6894 | Regression_eta: 0.7575 | Regression_pt: 1.9302 | Regression_sin_phi: 1.7923 | LR: 2.03e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.5888s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.1740 | max_loss: 0.0036 | max_backward: 0.1135 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 1.3124 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 5606.5 | uss_MB: 4766.5 | child_rss_MB: 8943.1 | allocated_MB: 608.5 | reserved_MB: 29990.0 | inactive_split_MB: 246.4 +rank1 [2026-10-27 19:52:28,508] INFO : Step 54000/100000 rank1 | Train Loss: 13.6142 | Train Loss Unweighted: 4.3355 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.3742 | Regression_cos_phi: 0.0499 | Regression_energy: 0.4083 | Regression_eta: 0.0736 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4652 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4567 | Classification_binary: 3.2885 | Regression_cos_phi: 1.6322 | Regression_energy: 1.7350 | Regression_eta: 0.9238 | Regression_pt: 2.0899 | Regression_sin_phi: 1.8711 | LR: 1.97e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4446s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1029 | max_loss: 0.0156 | max_backward: 0.1263 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2459 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 5600.5 | uss_MB: 4766.6 | child_rss_MB: 8886.2 | allocated_MB: 608.0 | reserved_MB: 29990.0 | inactive_split_MB: 249.1 +rank1 [2026-10-27 19:56:02,481] INFO : Step 55000/100000 rank1 | Train Loss: 14.4463 | Train Loss Unweighted: 4.6356 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5250 | Regression_cos_phi: 0.0497 | Regression_energy: 0.4959 | Regression_eta: 0.0660 | Regression_pt: 0.4403 | Regression_sin_phi: 0.0455 | Group Train Loss Unweighted (window mean): physical_pu: 3.6517 | physical_nopu: 4.4554 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3218 | Classification_binary: 3.4718 | Regression_cos_phi: 1.5198 | Regression_energy: 2.8120 | Regression_eta: 0.8574 | Regression_pt: 3.4142 | Regression_sin_phi: 1.6361 | LR: 1.92e-04 | DataLoad Time: 0.0027s | Model Forward Time: 2.9797s | Phase Times: max_pre_step_barrier: 0.0108 | max_batch_to_device: 0.0008 | max_forward: 0.0923 | max_loss: 0.0219 | max_backward: 0.1306 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 1.7796 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5599.8 | uss_MB: 4765.5 | child_rss_MB: 8883.4 | allocated_MB: 610.0 | reserved_MB: 29990.0 | inactive_split_MB: 274.0 +rank1 [2026-10-27 19:59:40,797] INFO : Step 56000/100000 rank1 | Train Loss: 12.4091 | Train Loss Unweighted: 4.2572 | Train Loss Components: Classification: 0.0135 | Classification_binary: 3.4523 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3421 | Regression_eta: 0.0664 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6493 | physical_nopu: 4.4547 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3310 | Classification_binary: 3.3038 | Regression_cos_phi: 1.2695 | Regression_energy: 1.9877 | Regression_eta: 0.9229 | Regression_pt: 2.2747 | Regression_sin_phi: 1.8955 | LR: 1.86e-04 | DataLoad Time: 0.3286s | Model Forward Time: 2.7411s | Phase Times: max_pre_step_barrier: 0.3290 | max_batch_to_device: 0.0902 | max_forward: 0.1771 | max_loss: 0.0038 | max_backward: 0.1058 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 1.4273 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5599.8 | uss_MB: 4766.9 | child_rss_MB: 8880.9 | allocated_MB: 608.1 | reserved_MB: 29990.0 | inactive_split_MB: 253.9 +rank1 [2026-10-27 20:03:29,496] INFO : Step 57000/100000 rank1 | Train Loss: 14.9087 | Train Loss Unweighted: 4.5957 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4371 | Regression_cos_phi: 0.0471 | Regression_energy: 0.5028 | Regression_eta: 0.0770 | Regression_pt: 0.4658 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6475 | physical_nopu: 4.4509 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5590 | Classification_binary: 3.5336 | Regression_cos_phi: 1.6665 | Regression_energy: 3.0341 | Regression_eta: 0.8717 | Regression_pt: 3.7211 | Regression_sin_phi: 2.5817 | LR: 1.81e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.4959s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1036 | max_loss: 0.0876 | max_backward: 0.2130 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 1.1951 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5597.8 | uss_MB: 4497.3 | child_rss_MB: 8866.5 | allocated_MB: 608.1 | reserved_MB: 30358.0 | inactive_split_MB: 283.9 +rank1 [2026-10-27 20:07:08,735] INFO : Step 58000/100000 rank1 | Train Loss: 12.4468 | Train Loss Unweighted: 4.2826 | Train Loss Components: Classification: 0.0105 | Classification_binary: 3.5299 | Regression_cos_phi: 0.0545 | Regression_energy: 0.3106 | Regression_eta: 0.0620 | Regression_pt: 0.2643 | Regression_sin_phi: 0.0509 | Group Train Loss Unweighted (window mean): physical_pu: 3.6440 | physical_nopu: 4.4492 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2650 | Classification_binary: 3.5476 | Regression_cos_phi: 2.1126 | Regression_energy: 1.4085 | Regression_eta: 0.8335 | Regression_pt: 1.5720 | Regression_sin_phi: 1.7097 | LR: 1.75e-04 | DataLoad Time: 0.0026s | Model Forward Time: 2.5426s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1868 | max_loss: 0.0037 | max_backward: 0.1111 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2860 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5587.6 | uss_MB: 3611.2 | child_rss_MB: 8803.1 | allocated_MB: 608.2 | reserved_MB: 30358.0 | inactive_split_MB: 337.0 +rank1 [2026-10-27 20:10:52,443] INFO : Step 59000/100000 rank1 | Train Loss: 13.1870 | Train Loss Unweighted: 4.3659 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4760 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3771 | Regression_eta: 0.0773 | Regression_pt: 0.3247 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4440 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5319 | Classification_binary: 3.5151 | Regression_cos_phi: 1.6217 | Regression_energy: 1.8662 | Regression_eta: 2.0031 | Regression_pt: 1.9855 | Regression_sin_phi: 2.0201 | LR: 1.70e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.4660s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1005 | max_loss: 0.0036 | max_backward: 0.1062 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 1.2390 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 5582.6 | uss_MB: 4766.7 | child_rss_MB: 8755.9 | allocated_MB: 610.1 | reserved_MB: 31234.0 | inactive_split_MB: 273.9 +rank1 [2026-10-27 20:14:34,818] INFO : Step 60000/100000 rank1 | Train Loss: 15.4000 | Train Loss Unweighted: 4.6905 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.3905 | Regression_cos_phi: 0.0456 | Regression_energy: 0.5805 | Regression_eta: 0.0703 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6406 | physical_nopu: 4.4523 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4676 | Classification_binary: 3.4652 | Regression_cos_phi: 1.8203 | Regression_energy: 2.6886 | Regression_eta: 1.3340 | Regression_pt: 3.2129 | Regression_sin_phi: 1.8949 | LR: 1.64e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.5445s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0757 | max_forward: 0.1624 | max_loss: 0.0040 | max_backward: 0.1315 | max_optimizer_step: 0.2155 | max_tensorboard_logging: 1.2531 | max_tensorboard_flush: 0.0020 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5582.8 | uss_MB: 4766.6 | child_rss_MB: 8760.5 | allocated_MB: 611.6 | reserved_MB: 31234.0 | inactive_split_MB: 318.4 +rank1 [2026-10-27 20:14:35,735] INFO : Running validation on rank1 +rank1 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 20:16:30,817] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 20:16:31,258] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 20:16:55,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 20:16:56,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 20:16:56,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 20:16:57,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 20:16:57,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 20:16:58,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 20:16:58,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 20:16:59,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 20:17:09,169] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 20:17:09,353] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 20:17:09,707] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 20:17:24,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 20:17:25,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 20:17:25,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 20:17:26,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 20:17:26,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 20:17:27,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 20:17:27,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 20:17:28,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 20:17:38,229] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 20:17:39,020] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 20:17:39,452] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 20:17:54,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 20:17:55,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 20:17:55,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 20:17:56,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 20:17:57,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 20:17:57,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 20:17:57,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 20:17:58,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 20:18:08,499] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 20:22:28,767] INFO : Step 61000/100000 rank1 | Train Loss: 12.1474 | Train Loss Unweighted: 4.3289 | Train Loss Components: Classification: 0.0110 | Classification_binary: 3.6236 | Regression_cos_phi: 0.0523 | Regression_energy: 0.2909 | Regression_eta: 0.0654 | Regression_pt: 0.2368 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.6358 | physical_nopu: 4.4439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3388 | Classification_binary: 3.5646 | Regression_cos_phi: 1.8574 | Regression_energy: 1.5047 | Regression_eta: 0.9069 | Regression_pt: 1.5529 | Regression_sin_phi: 1.9275 | LR: 1.59e-04 | DataLoad Time: 0.0057s | Model Forward Time: 3.4748s | Phase Times: max_pre_step_barrier: 0.0115 | max_batch_to_device: 0.0009 | max_forward: 0.2092 | max_loss: 0.0037 | max_backward: 0.1501 | max_optimizer_step: 0.2965 | max_tensorboard_logging: 1.8725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6817.2 | uss_MB: 6009.9 | child_rss_MB: 8755.0 | allocated_MB: 607.8 | reserved_MB: 28822.0 | inactive_split_MB: 222.9 +rank1 [2026-10-27 20:26:14,158] INFO : Step 62000/100000 rank1 | Train Loss: 11.7977 | Train Loss Unweighted: 3.6294 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9084 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2778 | Regression_eta: 0.1039 | Regression_pt: 0.2286 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6417 | physical_nopu: 4.4459 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4844 | Classification_binary: 2.9283 | Regression_cos_phi: 1.7560 | Regression_energy: 1.4464 | Regression_eta: 1.7188 | Regression_pt: 1.5375 | Regression_sin_phi: 1.8334 | LR: 1.54e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.7430s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0033 | max_forward: 0.2219 | max_loss: 0.0037 | max_backward: 0.2965 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1292 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1278 | Memory: rss_MB: 6812.4 | uss_MB: 6017.7 | child_rss_MB: 8760.3 | allocated_MB: 618.8 | reserved_MB: 28822.0 | inactive_split_MB: 888.0 +rank1 [2026-10-27 20:30:05,877] INFO : Step 63000/100000 rank1 | Train Loss: 12.5071 | Train Loss Unweighted: 4.3613 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5997 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3200 | Regression_eta: 0.0719 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6399 | physical_nopu: 4.4447 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3756 | Classification_binary: 3.5894 | Regression_cos_phi: 1.8524 | Regression_energy: 1.5377 | Regression_eta: 0.8997 | Regression_pt: 1.6816 | Regression_sin_phi: 1.8794 | LR: 1.49e-04 | DataLoad Time: 0.0010s | Model Forward Time: 11.0671s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1169 | max_loss: 0.0157 | max_backward: 0.2882 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 9.0944 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6806.2 | uss_MB: 4694.0 | child_rss_MB: 8747.4 | allocated_MB: 610.4 | reserved_MB: 28822.0 | inactive_split_MB: 261.6 +rank1 [2026-10-27 20:33:53,442] INFO : Step 64000/100000 rank1 | Train Loss: 14.0388 | Train Loss Unweighted: 4.5035 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.4200 | Regression_cos_phi: 0.0430 | Regression_energy: 0.4752 | Regression_eta: 0.0663 | Regression_pt: 0.4376 | Regression_sin_phi: 0.0442 | Group Train Loss Unweighted (window mean): physical_pu: 3.6357 | physical_nopu: 4.4395 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4852 | Classification_binary: 3.6541 | Regression_cos_phi: 1.7271 | Regression_energy: 3.1106 | Regression_eta: 0.9385 | Regression_pt: 3.8943 | Regression_sin_phi: 1.5856 | LR: 1.43e-04 | DataLoad Time: 0.0024s | Model Forward Time: 6.4866s | Phase Times: max_pre_step_barrier: 0.1731 | max_batch_to_device: 0.0118 | max_forward: 0.1219 | max_loss: 0.0038 | max_backward: 0.1121 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 5.0441 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5600.0 | uss_MB: 4820.3 | child_rss_MB: 8744.3 | allocated_MB: 611.1 | reserved_MB: 29052.0 | inactive_split_MB: 372.9 +rank1 [2026-10-27 20:37:48,515] INFO : Step 65000/100000 rank1 | Train Loss: 16.3374 | Train Loss Unweighted: 4.8919 | Train Loss Components: Classification: 0.0142 | Classification_binary: 3.4871 | Regression_cos_phi: 0.0500 | Regression_energy: 0.6360 | Regression_eta: 0.0628 | Regression_pt: 0.5943 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4335 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4679 | Classification_binary: 3.6267 | Regression_cos_phi: 1.5335 | Regression_energy: 2.7169 | Regression_eta: 0.7592 | Regression_pt: 3.3116 | Regression_sin_phi: 1.6855 | LR: 1.38e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.4884s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.2167 | max_loss: 0.0058 | max_backward: 0.2407 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 1.9060 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5594.0 | uss_MB: 4820.8 | child_rss_MB: 8750.8 | allocated_MB: 609.7 | reserved_MB: 29052.0 | inactive_split_MB: 236.9 +rank1 [2026-10-27 20:41:37,390] INFO : Step 66000/100000 rank1 | Train Loss: 12.8848 | Train Loss Unweighted: 4.2935 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4715 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3451 | Regression_eta: 0.0693 | Regression_pt: 0.2907 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.6349 | physical_nopu: 4.4348 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5821 | Classification_binary: 3.3774 | Regression_cos_phi: 1.6715 | Regression_energy: 2.9253 | Regression_eta: 1.5148 | Regression_pt: 3.1103 | Regression_sin_phi: 2.0374 | LR: 1.33e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.1922s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0854 | max_loss: 0.1083 | max_backward: 0.2178 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.9218 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5582.1 | uss_MB: 4820.7 | child_rss_MB: 8750.8 | allocated_MB: 611.4 | reserved_MB: 29052.0 | inactive_split_MB: 254.2 +rank1 [2026-10-27 20:45:24,537] INFO : Step 67000/100000 rank1 | Train Loss: 14.2125 | Train Loss Unweighted: 4.5026 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.4493 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4566 | Regression_eta: 0.0671 | Regression_pt: 0.4150 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6352 | physical_nopu: 4.4350 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4817 | Classification_binary: 3.4983 | Regression_cos_phi: 1.5550 | Regression_energy: 2.8466 | Regression_eta: 1.0153 | Regression_pt: 3.5528 | Regression_sin_phi: 2.3879 | LR: 1.28e-04 | DataLoad Time: 0.0028s | Model Forward Time: 3.0117s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0901 | max_loss: 0.0487 | max_backward: 0.1449 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 1.8090 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5575.1 | uss_MB: 4820.8 | child_rss_MB: 8750.2 | allocated_MB: 607.1 | reserved_MB: 29260.0 | inactive_split_MB: 246.6 +rank1 [2026-10-27 20:49:05,126] INFO : Step 68000/100000 rank1 | Train Loss: 12.0487 | Train Loss Unweighted: 4.2633 | Train Loss Components: Classification: 0.0107 | Classification_binary: 3.5135 | Regression_cos_phi: 0.0466 | Regression_energy: 0.3165 | Regression_eta: 0.0645 | Regression_pt: 0.2647 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4387 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4006 | Classification_binary: 3.4398 | Regression_cos_phi: 1.2951 | Regression_energy: 1.5798 | Regression_eta: 0.8981 | Regression_pt: 1.7652 | Regression_sin_phi: 1.4852 | LR: 1.24e-04 | DataLoad Time: 0.0026s | Model Forward Time: 2.8889s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0010 | max_forward: 0.0879 | max_loss: 0.0120 | max_backward: 0.1069 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.7176 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5570.9 | uss_MB: 4820.5 | child_rss_MB: 8753.3 | allocated_MB: 609.5 | reserved_MB: 29260.0 | inactive_split_MB: 225.9 +rank1 [2026-10-27 20:52:47,283] INFO : Step 69000/100000 rank1 | Train Loss: 13.1377 | Train Loss Unweighted: 4.3133 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4496 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3632 | Regression_eta: 0.0778 | Regression_pt: 0.3079 | Regression_sin_phi: 0.0541 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4321 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4089 | Classification_binary: 3.4486 | Regression_cos_phi: 1.5081 | Regression_energy: 2.0410 | Regression_eta: 0.7288 | Regression_pt: 2.2186 | Regression_sin_phi: 1.8804 | LR: 1.19e-04 | DataLoad Time: 0.0007s | Model Forward Time: 4.9135s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.1207 | max_loss: 0.0509 | max_backward: 0.2012 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 3.5532 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 5571.9 | uss_MB: 4820.8 | child_rss_MB: 8755.8 | allocated_MB: 608.0 | reserved_MB: 29260.0 | inactive_split_MB: 256.4 +rank1 [2026-10-27 20:56:27,513] INFO : Step 70000/100000 rank1 | Train Loss: 15.8477 | Train Loss Unweighted: 4.7317 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3873 | Regression_cos_phi: 0.0498 | Regression_energy: 0.6015 | Regression_eta: 0.0629 | Regression_pt: 0.5683 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6315 | physical_nopu: 4.4280 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5118 | Classification_binary: 3.3170 | Regression_cos_phi: 1.3642 | Regression_energy: 1.9344 | Regression_eta: 0.9815 | Regression_pt: 2.3646 | Regression_sin_phi: 1.4029 | LR: 1.14e-04 | DataLoad Time: 0.0060s | Model Forward Time: 2.9424s | Phase Times: max_pre_step_barrier: 0.0198 | max_batch_to_device: 0.0007 | max_forward: 0.1178 | max_loss: 0.0036 | max_backward: 0.1081 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.7610 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5571.5 | uss_MB: 4819.7 | child_rss_MB: 8751.0 | allocated_MB: 610.3 | reserved_MB: 30150.0 | inactive_split_MB: 260.3 +rank1 [2026-10-27 20:56:28,457] INFO : Running validation on rank1 +rank1 [2026-10-27 20:56:28,457] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 20:56:28,457] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 20:56:28,457] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 20:58:25,954] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 20:58:26,359] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 20:58:51,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 20:58:52,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 20:58:52,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 20:58:53,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 20:58:54,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 20:58:54,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 20:58:55,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 20:58:55,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 20:59:05,548] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 20:59:05,946] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 20:59:06,379] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 20:59:21,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 20:59:21,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 20:59:22,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 20:59:22,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 20:59:23,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 20:59:24,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 20:59:24,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 20:59:25,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 20:59:35,067] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 20:59:35,861] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 20:59:36,279] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 20:59:51,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 20:59:51,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 20:59:52,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 20:59:52,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 20:59:53,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 20:59:53,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 20:59:54,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 20:59:54,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 21:00:04,916] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 21:04:25,115] INFO : Step 71000/100000 rank1 | Train Loss: 12.0110 | Train Loss Unweighted: 3.6539 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9046 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2881 | Regression_eta: 0.1063 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6306 | physical_nopu: 4.4256 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4970 | Classification_binary: 2.9484 | Regression_cos_phi: 1.8260 | Regression_energy: 1.6780 | Regression_eta: 1.6106 | Regression_pt: 1.9088 | Regression_sin_phi: 1.9319 | LR: 1.10e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.4584s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0035 | max_forward: 0.1952 | max_loss: 0.0038 | max_backward: 0.3195 | max_optimizer_step: 0.3092 | max_tensorboard_logging: 1.7411 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6798.7 | uss_MB: 6058.3 | child_rss_MB: 8722.6 | allocated_MB: 623.5 | reserved_MB: 27422.0 | inactive_split_MB: 864.2 +rank1 [2026-10-27 21:08:01,993] INFO : Step 72000/100000 rank1 | Train Loss: 15.2878 | Train Loss Unweighted: 4.6447 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.3643 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5735 | Regression_eta: 0.0647 | Regression_pt: 0.5353 | Regression_sin_phi: 0.0449 | Group Train Loss Unweighted (window mean): physical_pu: 3.6275 | physical_nopu: 4.4239 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3898 | Classification_binary: 3.3928 | Regression_cos_phi: 1.8878 | Regression_energy: 1.8495 | Regression_eta: 1.2488 | Regression_pt: 2.0541 | Regression_sin_phi: 2.1644 | LR: 1.05e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.5451s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0232 | max_forward: 0.1174 | max_loss: 0.0036 | max_backward: 0.0921 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 1.2606 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5641.1 | uss_MB: 4896.7 | child_rss_MB: 8682.2 | allocated_MB: 607.6 | reserved_MB: 28586.0 | inactive_split_MB: 303.9 +rank1 [2026-10-27 21:11:50,556] INFO : Step 73000/100000 rank1 | Train Loss: 12.7380 | Train Loss Unweighted: 4.2392 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.4842 | Regression_cos_phi: 0.0575 | Regression_energy: 0.3093 | Regression_eta: 0.0643 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.6309 | physical_nopu: 4.4270 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4192 | Classification_binary: 3.4468 | Regression_cos_phi: 2.0526 | Regression_energy: 1.6770 | Regression_eta: 1.1734 | Regression_pt: 1.7410 | Regression_sin_phi: 2.2247 | LR: 1.01e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4473s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0985 | max_loss: 0.0632 | max_backward: 0.1654 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2139 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5637.1 | uss_MB: 4897.0 | child_rss_MB: 8681.3 | allocated_MB: 607.0 | reserved_MB: 28586.0 | inactive_split_MB: 227.5 +rank1 [2026-10-27 21:15:34,267] INFO : Step 74000/100000 rank1 | Train Loss: 12.5735 | Train Loss Unweighted: 4.2478 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4076 | Regression_cos_phi: 0.0429 | Regression_energy: 0.3533 | Regression_eta: 0.0699 | Regression_pt: 0.3116 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6307 | physical_nopu: 4.4218 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5028 | Classification_binary: 3.5776 | Regression_cos_phi: 1.8024 | Regression_energy: 2.0052 | Regression_eta: 1.0337 | Regression_pt: 2.4043 | Regression_sin_phi: 1.5788 | LR: 9.68e-05 | DataLoad Time: 0.0021s | Model Forward Time: 2.5464s | Phase Times: max_pre_step_barrier: 0.1687 | max_batch_to_device: 0.0008 | max_forward: 0.0990 | max_loss: 0.0516 | max_backward: 0.1766 | max_optimizer_step: 0.1812 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5626.2 | uss_MB: 4897.3 | child_rss_MB: 8634.5 | allocated_MB: 608.6 | reserved_MB: 28586.0 | inactive_split_MB: 285.4 +rank1 [2026-10-27 21:19:21,651] INFO : Step 75000/100000 rank1 | Train Loss: 15.1687 | Train Loss Unweighted: 4.6098 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.3751 | Regression_cos_phi: 0.0492 | Regression_energy: 0.5530 | Regression_eta: 0.0638 | Regression_pt: 0.5060 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4206 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4218 | Classification_binary: 3.4106 | Regression_cos_phi: 1.8309 | Regression_energy: 1.9858 | Regression_eta: 0.9354 | Regression_pt: 2.3680 | Regression_sin_phi: 2.0248 | LR: 9.27e-05 | DataLoad Time: 0.0022s | Model Forward Time: 2.4835s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1026 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.2820 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5619.2 | uss_MB: 4896.7 | child_rss_MB: 8602.7 | allocated_MB: 610.1 | reserved_MB: 28586.0 | inactive_split_MB: 259.7 +rank1 [2026-10-27 21:23:05,981] INFO : Step 76000/100000 rank1 | Train Loss: 12.2943 | Train Loss Unweighted: 4.2773 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5085 | Regression_cos_phi: 0.0476 | Regression_energy: 0.3262 | Regression_eta: 0.0732 | Regression_pt: 0.2631 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6289 | physical_nopu: 4.4209 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3518 | Classification_binary: 3.4251 | Regression_cos_phi: 1.4004 | Regression_energy: 1.9116 | Regression_eta: 0.8993 | Regression_pt: 2.1650 | Regression_sin_phi: 1.7302 | LR: 8.88e-05 | DataLoad Time: 0.0021s | Model Forward Time: 2.6228s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.1085 | max_optimizer_step: 0.1831 | max_tensorboard_logging: 1.4262 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 5617.3 | uss_MB: 4895.6 | child_rss_MB: 8601.6 | allocated_MB: 608.9 | reserved_MB: 28942.0 | inactive_split_MB: 227.1 +rank1 [2026-10-27 21:27:01,963] INFO : Step 77000/100000 rank1 | Train Loss: 13.8477 | Train Loss Unweighted: 4.4496 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4225 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4510 | Regression_eta: 0.0623 | Regression_pt: 0.4032 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6274 | physical_nopu: 4.4166 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5700 | Classification_binary: 3.4885 | Regression_cos_phi: 1.8324 | Regression_energy: 2.4657 | Regression_eta: 1.0447 | Regression_pt: 2.9194 | Regression_sin_phi: 1.6213 | LR: 8.50e-05 | DataLoad Time: 0.0019s | Model Forward Time: 6.1303s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0839 | max_forward: 0.1754 | max_loss: 0.0036 | max_backward: 0.1103 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 4.6772 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1374 | Memory: rss_MB: 5607.3 | uss_MB: 4838.5 | child_rss_MB: 8502.5 | allocated_MB: 611.0 | reserved_MB: 28942.0 | inactive_split_MB: 369.0 +rank1 [2026-10-27 21:30:52,663] INFO : Step 78000/100000 rank1 | Train Loss: 14.1981 | Train Loss Unweighted: 4.6212 | Train Loss Components: Classification: 0.0133 | Classification_binary: 3.5747 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4573 | Regression_eta: 0.0669 | Regression_pt: 0.4114 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.6241 | physical_nopu: 4.4230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4277 | Classification_binary: 3.6330 | Regression_cos_phi: 1.7345 | Regression_energy: 1.8600 | Regression_eta: 1.3415 | Regression_pt: 2.0517 | Regression_sin_phi: 2.5895 | LR: 8.13e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.0989s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1149 | max_loss: 0.0636 | max_backward: 0.1914 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5607.3 | uss_MB: 4897.1 | child_rss_MB: 8495.1 | allocated_MB: 606.6 | reserved_MB: 28942.0 | inactive_split_MB: 309.2 +rank1 [2026-10-27 21:34:49,721] INFO : Step 79000/100000 rank1 | Train Loss: 12.3648 | Train Loss Unweighted: 4.2396 | Train Loss Components: Classification: 0.0145 | Classification_binary: 3.4636 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3226 | Regression_eta: 0.0628 | Regression_pt: 0.2792 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6254 | physical_nopu: 4.4183 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4831 | Classification_binary: 3.3400 | Regression_cos_phi: 1.7190 | Regression_energy: 1.6233 | Regression_eta: 0.9141 | Regression_pt: 1.9512 | Regression_sin_phi: 1.2314 | LR: 7.78e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.3388s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0936 | max_forward: 0.1936 | max_loss: 0.0037 | max_backward: 0.1134 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 2.0079 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 5602.4 | uss_MB: 4897.2 | child_rss_MB: 8483.1 | allocated_MB: 609.2 | reserved_MB: 29222.0 | inactive_split_MB: 324.6 +rank1 [2026-10-27 21:38:37,320] INFO : Step 80000/100000 rank1 | Train Loss: 16.1833 | Train Loss Unweighted: 4.8046 | Train Loss Components: Classification: 0.0155 | Classification_binary: 3.4149 | Regression_cos_phi: 0.0453 | Regression_energy: 0.6189 | Regression_eta: 0.0801 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6233 | physical_nopu: 4.4142 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4894 | Classification_binary: 3.4911 | Regression_cos_phi: 1.7334 | Regression_energy: 2.6625 | Regression_eta: 1.3374 | Regression_pt: 3.4522 | Regression_sin_phi: 2.0384 | LR: 7.44e-05 | DataLoad Time: 0.0103s | Model Forward Time: 3.9234s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0008 | max_forward: 0.1071 | max_loss: 0.0037 | max_backward: 0.1380 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 2.6784 | max_tensorboard_flush: 0.0026 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 5596.4 | uss_MB: 4871.2 | child_rss_MB: 8436.2 | allocated_MB: 606.3 | reserved_MB: 29222.0 | inactive_split_MB: 471.7 +rank1 [2026-10-27 21:38:38,322] INFO : Running validation on rank1 +rank1 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 21:40:39,510] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 21:40:39,981] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 21:41:05,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 21:41:06,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 21:41:06,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 21:41:07,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 21:41:07,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 21:41:08,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 21:41:08,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 21:41:09,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 21:41:19,167] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 21:41:19,914] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 21:41:20,297] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 21:41:35,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 21:41:36,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 21:41:36,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 21:41:37,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 21:41:37,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 21:41:38,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 21:41:38,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 21:41:39,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 21:41:49,450] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-27 21:41:49,545] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 21:41:50,006] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 21:42:05,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 21:42:05,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 21:42:06,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 21:42:06,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 21:42:07,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 21:42:07,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 21:42:08,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 21:42:08,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 21:42:18,680] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 21:46:50,145] INFO : Step 81000/100000 rank1 | Train Loss: 11.7500 | Train Loss Unweighted: 3.6065 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.8934 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2712 | Regression_eta: 0.1074 | Regression_pt: 0.2244 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4130 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4554 | Classification_binary: 2.8828 | Regression_cos_phi: 1.6991 | Regression_energy: 1.4380 | Regression_eta: 1.5720 | Regression_pt: 1.5604 | Regression_sin_phi: 1.8557 | LR: 7.11e-05 | DataLoad Time: 0.0016s | Model Forward Time: 8.6910s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0975 | max_forward: 0.3237 | max_loss: 0.0036 | max_backward: 0.5496 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 6.3197 | max_tensorboard_flush: 0.0011 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6963.5 | uss_MB: 4835.8 | child_rss_MB: 8415.8 | allocated_MB: 619.9 | reserved_MB: 28598.0 | inactive_split_MB: 1192.4 +rank1 [2026-10-27 21:50:33,876] INFO : Step 82000/100000 rank1 | Train Loss: 13.6704 | Train Loss Unweighted: 4.5604 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0419 | Regression_energy: 0.4618 | Regression_eta: 0.0599 | Regression_pt: 0.4291 | Regression_sin_phi: 0.0407 | Group Train Loss Unweighted (window mean): physical_pu: 3.6250 | physical_nopu: 4.4152 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4556 | Classification_binary: 3.6144 | Regression_cos_phi: 1.5227 | Regression_energy: 2.8217 | Regression_eta: 0.9951 | Regression_pt: 3.5911 | Regression_sin_phi: 1.8235 | LR: 6.80e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.1346s | Phase Times: max_pre_step_barrier: 0.1835 | max_batch_to_device: 0.0874 | max_forward: 0.1976 | max_loss: 0.0036 | max_backward: 0.1170 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 1.6260 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 5659.6 | uss_MB: 4976.9 | child_rss_MB: 8356.3 | allocated_MB: 607.5 | reserved_MB: 28598.0 | inactive_split_MB: 253.8 +rank1 [2026-10-27 21:54:19,227] INFO : Step 83000/100000 rank1 | Train Loss: 12.0270 | Train Loss Unweighted: 4.2414 | Train Loss Components: Classification: 0.0092 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0499 | Regression_energy: 0.3056 | Regression_eta: 0.0671 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4141 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2260 | Classification_binary: 3.3766 | Regression_cos_phi: 1.6799 | Regression_energy: 1.4293 | Regression_eta: 0.8055 | Regression_pt: 1.5019 | Regression_sin_phi: 1.6155 | LR: 6.51e-05 | DataLoad Time: 0.0013s | Model Forward Time: 3.7072s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1966 | max_loss: 0.0138 | max_backward: 0.1124 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 2.4026 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 5659.6 | uss_MB: 4977.1 | child_rss_MB: 8350.2 | allocated_MB: 609.6 | reserved_MB: 28598.0 | inactive_split_MB: 289.5 +rank1 [2026-10-27 21:58:08,515] INFO : Step 84000/100000 rank1 | Train Loss: 12.5172 | Train Loss Unweighted: 4.2412 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.4173 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3464 | Regression_eta: 0.0637 | Regression_pt: 0.3052 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4149 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4830 | Classification_binary: 3.2926 | Regression_cos_phi: 1.6137 | Regression_energy: 2.1499 | Regression_eta: 0.7043 | Regression_pt: 2.5455 | Regression_sin_phi: 1.8272 | LR: 6.23e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.2649s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1103 | max_loss: 0.0037 | max_backward: 0.1159 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 2.0319 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5655.6 | uss_MB: 4977.8 | child_rss_MB: 8312.0 | allocated_MB: 605.9 | reserved_MB: 28598.0 | inactive_split_MB: 208.8 +rank1 [2026-10-27 22:01:46,561] INFO : Step 85000/100000 rank1 | Train Loss: 15.2814 | Train Loss Unweighted: 4.6863 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.3811 | Regression_cos_phi: 0.0432 | Regression_energy: 0.5842 | Regression_eta: 0.0608 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6173 | physical_nopu: 4.4078 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5882 | Classification_binary: 3.5752 | Regression_cos_phi: 2.0053 | Regression_energy: 2.3591 | Regression_eta: 0.8828 | Regression_pt: 2.8065 | Regression_sin_phi: 2.0368 | LR: 5.96e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.0837s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.0993 | max_loss: 0.0558 | max_backward: 0.1762 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 1.8022 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 5645.3 | uss_MB: 4977.4 | child_rss_MB: 8242.1 | allocated_MB: 606.6 | reserved_MB: 28598.0 | inactive_split_MB: 257.4 +rank1 [2026-10-27 22:05:28,665] INFO : Step 86000/100000 rank1 | Train Loss: 12.8074 | Train Loss Unweighted: 4.3373 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.5785 | Regression_cos_phi: 0.0543 | Regression_energy: 0.3116 | Regression_eta: 0.0747 | Regression_pt: 0.2515 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.6255 | physical_nopu: 4.4086 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3595 | Classification_binary: 3.5043 | Regression_cos_phi: 1.9092 | Regression_energy: 1.6748 | Regression_eta: 1.1253 | Regression_pt: 1.8582 | Regression_sin_phi: 2.0315 | LR: 5.71e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.0433s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1215 | max_loss: 0.0037 | max_backward: 0.1229 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 1.6761 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5636.3 | uss_MB: 4977.9 | child_rss_MB: 8180.8 | allocated_MB: 608.3 | reserved_MB: 29688.0 | inactive_split_MB: 271.8 +rank1 [2026-10-27 22:09:22,619] INFO : Step 87000/100000 rank1 | Train Loss: 12.9093 | Train Loss Unweighted: 4.3003 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.3807 | Regression_cos_phi: 0.0447 | Regression_energy: 0.4014 | Regression_eta: 0.0615 | Regression_pt: 0.3523 | Regression_sin_phi: 0.0424 | Group Train Loss Unweighted (window mean): physical_pu: 3.6211 | physical_nopu: 4.4120 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5390 | Classification_binary: 3.5732 | Regression_cos_phi: 2.1026 | Regression_energy: 2.3685 | Regression_eta: 0.9037 | Regression_pt: 2.7667 | Regression_sin_phi: 2.0189 | LR: 5.48e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.0852s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0203 | max_forward: 0.1299 | max_loss: 0.0038 | max_backward: 0.1175 | max_optimizer_step: 0.2192 | max_tensorboard_logging: 1.8010 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5636.1 | uss_MB: 4575.9 | child_rss_MB: 8182.7 | allocated_MB: 607.0 | reserved_MB: 29688.0 | inactive_split_MB: 231.0 +rank1 [2026-10-27 22:13:02,799] INFO : Step 88000/100000 rank1 | Train Loss: 15.2369 | Train Loss Unweighted: 4.6456 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4525 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5240 | Regression_eta: 0.0665 | Regression_pt: 0.4840 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6192 | physical_nopu: 4.4119 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4492 | Classification_binary: 3.2578 | Regression_cos_phi: 1.4197 | Regression_energy: 3.2303 | Regression_eta: 0.8691 | Regression_pt: 4.0386 | Regression_sin_phi: 1.6417 | LR: 5.26e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.1164s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0007 | max_forward: 0.1127 | max_loss: 0.0236 | max_backward: 0.1542 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 1.7956 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5636.1 | uss_MB: 4977.3 | child_rss_MB: 8181.7 | allocated_MB: 609.0 | reserved_MB: 29688.0 | inactive_split_MB: 221.0 +rank1 [2026-10-27 22:16:52,919] INFO : Step 89000/100000 rank1 | Train Loss: 12.8609 | Train Loss Unweighted: 4.2765 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.4665 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3343 | Regression_eta: 0.0681 | Regression_pt: 0.2896 | Regression_sin_phi: 0.0555 | Group Train Loss Unweighted (window mean): physical_pu: 3.6177 | physical_nopu: 4.4110 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5002 | Classification_binary: 3.5604 | Regression_cos_phi: 2.2903 | Regression_energy: 2.0945 | Regression_eta: 0.9844 | Regression_pt: 2.3799 | Regression_sin_phi: 2.9247 | LR: 5.06e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.1961s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1194 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 1.9452 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 5632.5 | uss_MB: 4977.7 | child_rss_MB: 8137.7 | allocated_MB: 612.6 | reserved_MB: 30628.0 | inactive_split_MB: 309.7 +rank1 [2026-10-27 22:20:43,711] INFO : Step 90000/100000 rank1 | Train Loss: 11.6102 | Train Loss Unweighted: 3.6127 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9177 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2661 | Regression_eta: 0.1040 | Regression_pt: 0.2155 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4109 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4639 | Classification_binary: 2.9544 | Regression_cos_phi: 1.7514 | Regression_energy: 1.4154 | Regression_eta: 1.5525 | Regression_pt: 1.5440 | Regression_sin_phi: 1.8311 | LR: 4.88e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.5770s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0110 | max_forward: 0.1856 | max_loss: 0.0145 | max_backward: 0.4526 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 1.8620 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 5624.5 | uss_MB: 4985.7 | child_rss_MB: 8062.4 | allocated_MB: 617.2 | reserved_MB: 30628.0 | inactive_split_MB: 962.2 +rank1 [2026-10-27 22:20:44,689] INFO : Running validation on rank1 +rank1 [2026-10-27 22:20:44,689] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 22:22:45,204] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 22:22:45,606] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 22:23:10,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 22:23:10,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 22:23:11,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 22:23:11,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 22:23:12,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 22:23:12,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 22:23:13,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 22:23:13,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 22:23:23,925] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 22:23:24,365] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 22:23:24,734] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 22:23:40,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 22:23:40,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 22:23:41,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 22:23:41,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 22:23:42,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 22:23:42,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 22:23:43,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 22:23:43,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 22:23:53,797] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-27 22:23:53,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 22:23:54,274] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 22:24:09,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 22:24:09,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 22:24:10,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 22:24:10,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 22:24:11,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 22:24:11,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 22:24:12,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 22:24:12,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 22:24:22,978] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 22:28:41,211] INFO : Step 91000/100000 rank1 | Train Loss: 12.9063 | Train Loss Unweighted: 4.3371 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5236 | Regression_cos_phi: 0.0531 | Regression_energy: 0.3431 | Regression_eta: 0.0707 | Regression_pt: 0.2829 | Regression_sin_phi: 0.0512 | Group Train Loss Unweighted (window mean): physical_pu: 3.6180 | physical_nopu: 4.4087 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4374 | Classification_binary: 3.6335 | Regression_cos_phi: 2.4157 | Regression_energy: 1.9491 | Regression_eta: 1.8096 | Regression_pt: 2.0915 | Regression_sin_phi: 1.7276 | LR: 4.71e-05 | DataLoad Time: 0.0008s | Model Forward Time: 4.9011s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0055 | max_forward: 0.1521 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 3.4850 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7066.6 | uss_MB: 6432.9 | child_rss_MB: 8020.4 | allocated_MB: 607.8 | reserved_MB: 28796.0 | inactive_split_MB: 320.2 +rank1 [2026-10-27 22:32:23,431] INFO : Step 92000/100000 rank1 | Train Loss: 15.4341 | Train Loss Unweighted: 4.7247 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4285 | Regression_cos_phi: 0.0457 | Regression_energy: 0.5811 | Regression_eta: 0.0735 | Regression_pt: 0.5375 | Regression_sin_phi: 0.0425 | Group Train Loss Unweighted (window mean): physical_pu: 3.6150 | physical_nopu: 4.4032 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5836 | Classification_binary: 3.2955 | Regression_cos_phi: 1.7710 | Regression_energy: 3.2917 | Regression_eta: 1.6020 | Regression_pt: 4.1216 | Regression_sin_phi: 1.3647 | LR: 4.57e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.3948s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.2696 | max_loss: 0.0037 | max_backward: 0.1715 | max_optimizer_step: 0.3237 | max_tensorboard_logging: 1.7768 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7051.6 | uss_MB: 6432.9 | child_rss_MB: 7931.4 | allocated_MB: 609.3 | reserved_MB: 28796.0 | inactive_split_MB: 222.8 +rank1 [2026-10-27 22:36:08,796] INFO : Step 93000/100000 rank1 | Train Loss: 13.9744 | Train Loss Unweighted: 4.4854 | Train Loss Components: Classification: 0.0118 | Classification_binary: 3.5124 | Regression_cos_phi: 0.0578 | Regression_energy: 0.4245 | Regression_eta: 0.0652 | Regression_pt: 0.3624 | Regression_sin_phi: 0.0513 | Group Train Loss Unweighted (window mean): physical_pu: 3.6205 | physical_nopu: 4.4075 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3055 | Classification_binary: 3.6195 | Regression_cos_phi: 2.2058 | Regression_energy: 2.0141 | Regression_eta: 0.9309 | Regression_pt: 2.2112 | Regression_sin_phi: 1.9779 | LR: 4.43e-05 | DataLoad Time: 0.0018s | Model Forward Time: 2.9853s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1098 | max_loss: 0.0036 | max_backward: 0.1173 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 1.7709 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5706.9 | uss_MB: 5092.9 | child_rss_MB: 7894.0 | allocated_MB: 606.4 | reserved_MB: 30394.0 | inactive_split_MB: 244.2 +rank1 [2026-10-27 22:39:53,036] INFO : Step 94000/100000 rank1 | Train Loss: 12.4397 | Train Loss Unweighted: 4.2538 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4230 | Regression_cos_phi: 0.0415 | Regression_energy: 0.3513 | Regression_eta: 0.0702 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6164 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4426 | Classification_binary: 3.2892 | Regression_cos_phi: 1.5047 | Regression_energy: 2.1736 | Regression_eta: 1.2675 | Regression_pt: 2.3752 | Regression_sin_phi: 1.7041 | LR: 4.32e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.1514s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0009 | max_forward: 0.1540 | max_loss: 0.0036 | max_backward: 0.2309 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 1.7723 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 5695.9 | uss_MB: 4753.0 | child_rss_MB: 7806.9 | allocated_MB: 607.1 | reserved_MB: 30394.0 | inactive_split_MB: 232.2 +rank1 [2026-10-27 22:43:36,954] INFO : Step 95000/100000 rank1 | Train Loss: 14.9177 | Train Loss Unweighted: 4.6108 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3911 | Regression_cos_phi: 0.0436 | Regression_energy: 0.5612 | Regression_eta: 0.0598 | Regression_pt: 0.4905 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6121 | physical_nopu: 4.4076 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5660 | Classification_binary: 3.0690 | Regression_cos_phi: 1.4869 | Regression_energy: 3.7682 | Regression_eta: 0.7565 | Regression_pt: 4.7981 | Regression_sin_phi: 1.2242 | LR: 4.22e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.0355s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1111 | max_loss: 0.0199 | max_backward: 0.1395 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.7978 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5695.9 | uss_MB: 5093.1 | child_rss_MB: 7811.5 | allocated_MB: 609.0 | reserved_MB: 30394.0 | inactive_split_MB: 285.3 +rank1 [2026-10-27 22:47:14,355] INFO : Step 96000/100000 rank1 | Train Loss: 12.4738 | Train Loss Unweighted: 4.2244 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.4884 | Regression_cos_phi: 0.0540 | Regression_energy: 0.2967 | Regression_eta: 0.0804 | Regression_pt: 0.2429 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6137 | physical_nopu: 4.4081 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3041 | Classification_binary: 3.5026 | Regression_cos_phi: 1.8811 | Regression_energy: 1.7116 | Regression_eta: 1.6488 | Regression_pt: 1.9746 | Regression_sin_phi: 2.3329 | LR: 4.14e-05 | DataLoad Time: 0.0027s | Model Forward Time: 3.1601s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0007 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.2003 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 1.8675 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5695.9 | uss_MB: 5093.2 | child_rss_MB: 7804.9 | allocated_MB: 608.1 | reserved_MB: 30394.0 | inactive_split_MB: 259.4 +rank1 [2026-10-27 22:50:52,983] INFO : Step 97000/100000 rank1 | Train Loss: 12.7922 | Train Loss Unweighted: 4.2852 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.4355 | Regression_cos_phi: 0.0455 | Regression_energy: 0.3604 | Regression_eta: 0.0703 | Regression_pt: 0.3091 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6136 | physical_nopu: 4.4015 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5069 | Classification_binary: 3.6364 | Regression_cos_phi: 1.6028 | Regression_energy: 1.9882 | Regression_eta: 1.1006 | Regression_pt: 2.2305 | Regression_sin_phi: 1.7692 | LR: 4.08e-05 | DataLoad Time: 0.0026s | Model Forward Time: 3.0102s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1110 | max_loss: 0.0174 | max_backward: 0.1239 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 1.7282 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0908 | Memory: rss_MB: 5687.4 | uss_MB: 5092.4 | child_rss_MB: 7746.9 | allocated_MB: 607.2 | reserved_MB: 30394.0 | inactive_split_MB: 265.3 +rank1 [2026-10-27 22:54:49,599] INFO : Step 98000/100000 rank1 | Train Loss: 13.8508 | Train Loss Unweighted: 4.4512 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3736 | Regression_cos_phi: 0.0443 | Regression_energy: 0.4792 | Regression_eta: 0.0606 | Regression_pt: 0.4354 | Regression_sin_phi: 0.0433 | Group Train Loss Unweighted (window mean): physical_pu: 3.6169 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4276 | Classification_binary: 3.3116 | Regression_cos_phi: 1.3420 | Regression_energy: 2.1913 | Regression_eta: 0.7619 | Regression_pt: 2.8056 | Regression_sin_phi: 1.3649 | LR: 4.04e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.0654s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0119 | max_forward: 0.1317 | max_loss: 0.0036 | max_backward: 0.1246 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.7312 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0878 | Memory: rss_MB: 5687.4 | uss_MB: 5093.6 | child_rss_MB: 7731.7 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 292.7 +rank1 [2026-10-27 22:58:30,099] INFO : Step 99000/100000 rank1 | Train Loss: 12.5729 | Train Loss Unweighted: 4.3128 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5536 | Regression_cos_phi: 0.0519 | Regression_energy: 0.3097 | Regression_eta: 0.0644 | Regression_pt: 0.2679 | Regression_sin_phi: 0.0529 | Group Train Loss Unweighted (window mean): physical_pu: 3.6154 | physical_nopu: 4.4035 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2781 | Classification_binary: 3.6482 | Regression_cos_phi: 1.9503 | Regression_energy: 1.5337 | Regression_eta: 1.1595 | Regression_pt: 1.6672 | Regression_sin_phi: 2.1751 | LR: 4.01e-05 | DataLoad Time: 0.0116s | Model Forward Time: 3.0439s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0008 | max_forward: 0.1158 | max_loss: 0.0113 | max_backward: 0.1168 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 1.7726 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 5687.4 | uss_MB: 5093.5 | child_rss_MB: 7727.1 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 283.7 +rank1 [2026-10-27 23:02:17,099] INFO : Step 100000/100000 rank1 | Train Loss: 11.9488 | Train Loss Unweighted: 3.6450 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9057 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2836 | Regression_eta: 0.1038 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6201 | physical_nopu: 4.4070 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4673 | Classification_binary: 2.9079 | Regression_cos_phi: 1.7577 | Regression_energy: 1.4624 | Regression_eta: 1.4586 | Regression_pt: 1.6166 | Regression_sin_phi: 1.8800 | LR: 4.00e-05 | DataLoad Time: 0.0035s | Model Forward Time: 6.3799s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0031 | max_forward: 0.2860 | max_loss: 0.0036 | max_backward: 0.4897 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 4.4601 | max_tensorboard_flush: 0.0009 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5687.2 | uss_MB: 4556.1 | child_rss_MB: 7711.3 | allocated_MB: 618.6 | reserved_MB: 30394.0 | inactive_split_MB: 1112.1 +rank1 [2026-10-27 23:02:18,075] INFO : Running validation on rank1 +rank1 [2026-10-27 23:02:18,075] INFO : ResumableSampler physical_pu:valid resetting +rank1 [2026-10-27 23:02:18,075] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank1 [2026-10-27 23:02:18,075] INFO : ResumableSampler physical_nopu:valid resetting +rank1 [2026-10-27 23:02:18,075] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank1 [2026-10-27 23:04:23,623] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 23:04:24,087] INFO : Running predictions on cms_pf_ttbar +rank1 [2026-10-27 23:04:49,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_1_0.parquet +rank1 [2026-10-27 23:04:50,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_1_1.parquet +rank1 [2026-10-27 23:04:50,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_1_2.parquet +rank1 [2026-10-27 23:04:51,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_1_3.parquet +rank1 [2026-10-27 23:04:51,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_1_4.parquet +rank1 [2026-10-27 23:04:52,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_1_5.parquet +rank1 [2026-10-27 23:04:52,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_1_6.parquet +rank1 [2026-10-27 23:04:53,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_1_7.parquet +rank1 [2026-10-27 23:05:03,382] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 23:05:03,811] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 23:05:04,178] INFO : Running predictions on cms_pf_qcd +rank1 [2026-10-27 23:05:19,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_1_0.parquet +rank1 [2026-10-27 23:05:20,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_1_1.parquet +rank1 [2026-10-27 23:05:20,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_1_2.parquet +rank1 [2026-10-27 23:05:21,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_1_3.parquet +rank1 [2026-10-27 23:05:21,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_1_4.parquet +rank1 [2026-10-27 23:05:22,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_1_5.parquet +rank1 [2026-10-27 23:05:22,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_1_6.parquet +rank1 [2026-10-27 23:05:23,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_1_7.parquet +rank1 [2026-10-27 23:05:33,438] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-27 23:05:33,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-27 23:05:34,201] INFO : Running predictions on cms_pf_ztt +rank1 [2026-10-27 23:05:49,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_1_0.parquet +rank1 [2026-10-27 23:05:50,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_1_1.parquet +rank1 [2026-10-27 23:05:50,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_1_2.parquet +rank1 [2026-10-27 23:05:51,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_1_3.parquet +rank1 [2026-10-27 23:05:51,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_1_4.parquet +rank1 [2026-10-27 23:05:52,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_1_5.parquet +rank1 [2026-10-27 23:05:52,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_1_6.parquet +rank1 [2026-10-27 23:05:53,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_1_7.parquet +rank1 [2026-10-27 23:06:03,219] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-27 23:06:03,944] INFO : Training completed. Total time on device 1: 414.795min diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.2 b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.2 new file mode 100644 index 0000000000000000000000000000000000000000..b119c9a8cc906c81aed66240c7da2eb821e05268 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.2 @@ -0,0 +1,897 @@ +rank2 [2026-10-27 16:10:10,389] INFO : Initializing model with seed=12345; process seed=12347 +rank2 [2026-10-27 16:10:10,390] INFO : configured dtype=torch.bfloat16 for autocast +rank2 [2026-10-27 16:10:35,267] INFO : Instantiating model +rank2 [2026-10-27 16:10:35,267] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank2 [2026-10-27 16:10:35,267] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank2 [2026-10-27 16:10:35,296] INFO : Input encoding initialization took 0.03s +rank2 [2026-10-27 16:10:35,296] INFO : nn0 parameters: 2130944 +rank2 [2026-10-27 16:10:35,296] INFO : Initializing shared backbone layers, num_convs=3 +rank2 [2026-10-27 16:10:35,296] INFO : Initializing backbone layer 0 +rank2 [2026-10-27 16:10:35,350] INFO : Initializing backbone layer 1 +rank2 [2026-10-27 16:10:35,379] INFO : Initializing backbone layer 2 +rank2 [2026-10-27 16:10:35,407] INFO : Backbone initialization took 0.11s +rank2 [2026-10-27 16:10:35,407] INFO : backbone parameters: 15765696 +rank2 [2026-10-27 16:10:35,407] INFO : Initializing output DNNs +rank2 [2026-10-27 16:10:35,441] INFO : Output DNNs initialization took 0.03s +rank2 [2026-10-27 16:10:35,441] INFO : backbone_mode=BackboneMode.SHARED +rank2 [2026-10-27 16:10:35,441] INFO : jagged_attention enabled=False +rank2 [2026-10-27 16:10:35,441] INFO : flash_attn_varlen enabled=False +rank2 [2026-10-27 16:10:35,441] INFO : task_queries enabled=False +rank2 [2026-10-27 16:10:35,441] INFO : classification_norm parameters: 0 +rank2 [2026-10-27 16:10:35,441] INFO : regression_norm parameters: 0 +rank2 [2026-10-27 16:10:35,441] INFO : classification_query parameters: 0 +rank2 [2026-10-27 16:10:35,441] INFO : regression_query parameters: 0 +rank2 [2026-10-27 16:10:35,441] INFO : classification_readout parameters: 0 +rank2 [2026-10-27 16:10:35,441] INFO : regression_readout parameters: 0 +rank2 [2026-10-27 16:10:35,441] INFO : output_mode=elementwise +rank2 [2026-10-27 16:10:35,441] INFO : set_decoder parameters: 0 +rank2 [2026-10-27 16:10:35,441] INFO : nn_binary_particle parameters: 264706 +rank2 [2026-10-27 16:10:35,441] INFO : nn_pid parameters: 268297 +rank2 [2026-10-27 16:10:35,442] INFO : nn_pt parameters: 2113544 +rank2 [2026-10-27 16:10:35,442] INFO : nn_eta parameters: 264706 +rank2 [2026-10-27 16:10:35,442] INFO : nn_sin_phi parameters: 264706 +rank2 [2026-10-27 16:10:35,442] INFO : nn_cos_phi parameters: 264706 +rank2 [2026-10-27 16:10:35,442] INFO : nn_energy parameters: 2113544 +rank2 [2026-10-27 16:10:35,442] INFO : Total MLPF parameters: 23450849 +rank2 [2026-10-27 16:10:35,443] INFO : MLPF __init__ done +rank2 [2026-10-27 16:10:35,443] INFO : Instantiated model +rank2 [2026-10-27 16:10:35,443] INFO : Moving model to device rank=2 +rank2 [2026-10-27 16:10:35,468] INFO : Moved model to device rank=2 +rank2 [2026-10-27 16:10:35,469] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank2 [2026-10-27 16:10:35,470] INFO : Configured model for SyncBatchNorm rank=2 +rank2 [2026-10-27 16:10:36,003] INFO : Configured model for DistributedDataParallel rank=2 +rank2 [2026-10-27 16:10:36,123] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +202 module.nn_energy.nn.7.0.bias ... 0 +203 module.nn_energy.nn.7.2.weight ... 0 +204 module.nn_energy.nn.7.2.bias ... 0 +205 module.nn_energy.nn.7.4.weight ... 0 +206 module.nn_energy.nn.7.4.bias ... 0 + +[207 rows x 3 columns] +rank2 [2026-10-27 16:10:36,123] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x HEPTv2Layer( + embedding_dim=512, num_heads=16, width=512, dropout=0.0, block_size=100, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=512, out_features=512, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=512, out_features=512, bias=False) + (w_k): Linear(in_features=512, out_features=512, bias=False) + (w_v): Linear(in_features=512, out_features=512, bias=False) + (w_rpe): Linear(in_features=10, out_features=512, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=2048, bias=True) + (1): LayerNorm((2048,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=2048, out_features=512, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=512, out_features=2048, bias=False) + (up_proj): Linear(in_features=512, out_features=2048, bias=False) + (down_proj): Linear(in_features=2048, out_features=512, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank2 [2026-10-27 16:10:36,123] INFO : Trainable parameters: 23450849 +rank2 [2026-10-27 16:10:36,123] INFO : Non-trainable parameters: 22386 +rank2 [2026-10-27 16:10:36,124] INFO : Total parameters: 23473235 +rank2 [2026-10-27 16:10:36,126] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 262144 0 + module.backbone.0.attn.out_linear.bias 512 0 + module.backbone.0.attn.e2lsh.alpha 0 1632 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 262144 0 + module.backbone.0.w_k.weight 262144 0 + module.backbone.0.w_v.weight 262144 0 + module.backbone.0.w_rpe.weight 0 5120 + module.backbone.0.w_rpe.bias 0 512 +module.backbone.0.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm2.weight 512 0 + module.backbone.0.q_norm.weight 32 0 + module.backbone.0.k_norm.weight 32 0 + module.backbone.0.ff.gate_proj.weight 1048576 0 + module.backbone.0.ff.up_proj.weight 1048576 0 + module.backbone.0.ff.down_proj.weight 1048576 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 262144 0 + module.backbone.1.attn.out_linear.bias 512 0 + module.backbone.1.attn.e2lsh.alpha 0 1632 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 262144 0 + module.backbone.1.w_k.weight 262144 0 + module.backbone.1.w_v.weight 262144 0 + module.backbone.1.w_rpe.weight 0 5120 + module.backbone.1.w_rpe.bias 0 512 +module.backbone.1.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm2.weight 512 0 + module.backbone.1.q_norm.weight 32 0 + module.backbone.1.k_norm.weight 32 0 + module.backbone.1.ff.gate_proj.weight 1048576 0 + module.backbone.1.ff.up_proj.weight 1048576 0 + module.backbone.1.ff.down_proj.weight 1048576 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 262144 0 + module.backbone.2.attn.out_linear.bias 512 0 + module.backbone.2.attn.e2lsh.alpha 0 1632 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 262144 0 + module.backbone.2.w_k.weight 262144 0 + module.backbone.2.w_v.weight 262144 0 + module.backbone.2.w_rpe.weight 0 5120 + module.backbone.2.w_rpe.bias 0 512 +module.backbone.2.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm2.weight 512 0 + module.backbone.2.q_norm.weight 32 0 + module.backbone.2.k_norm.weight 32 0 + module.backbone.2.ff.gate_proj.weight 1048576 0 + module.backbone.2.ff.up_proj.weight 1048576 0 + module.backbone.2.ff.down_proj.weight 1048576 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank2 [2026-10-27 16:10:36,126] INFO : Getting dataloaders +rank2 [2026-10-27 16:10:36,126] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:43,313] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:44,211] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:44,829] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank2 [2026-10-27 16:10:44,829] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:46,047] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:46,991] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:47,896] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank2 [2026-10-27 16:10:47,896] INFO : Creating WeightedMixIterator with 2 data loaders. +rank2 [2026-10-27 16:10:47,896] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:48,051] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:48,209] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:48,357] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank2 [2026-10-27 16:10:48,358] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:48,531] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:48,710] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:10:48,850] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank2 [2026-10-27 16:10:48,850] INFO : Creating InterleavedIterator with 2 data loaders. +rank2 [2026-10-27 16:11:16,408] INFO : Got dataloaders +rank2 [2026-10-27 16:11:16,408] INFO : loader split=train rank=2 len=232058 +rank2 [2026-10-27 16:11:16,408] INFO : loader split=valid rank=2 len=48 +rank2 [2026-10-27 16:11:16,408] INFO : Starting training from step 1 to 100000 on rank 2 of 8 +rank2 [2026-10-27 16:11:16,408] INFO : Creating train_iterator +rank2 [2026-10-27 16:11:16,408] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank2 [2026-10-27 16:11:30,773] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank2 [2026-10-27 16:11:46,256] INFO : Created train_iterator +rank2 [2026-10-27 16:12:32,258] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.129867712855339, 'Classification': 0.16580097352620216, 'Regression_pt': 0.7168090478330851, 'Regression_eta': 0.335200086296536, 'Regression_sin_phi': 0.1274821354681626, 'Regression_cos_phi': 0.1371471309568733, 'Regression_energy': 0.9554258581250906}, weights={'Classification_binary': 1.0, 'Classification': 30.939912796020508, 'Regression_pt': 7.156533241271973, 'Regression_eta': 15.3038969039917, 'Regression_sin_phi': 40.23989486694336, 'Regression_cos_phi': 37.40412139892578, 'Regression_energy': 5.369194984436035} +rank2 [2026-10-27 16:15:18,322] INFO : Step 1000/100000 rank2 | Train Loss: 16.8632 | Train Loss Unweighted: 5.4934 | Train Loss Components: Classification: 0.0189 | Classification_binary: 4.1833 | Regression_cos_phi: 0.0509 | Regression_energy: 0.5806 | Regression_eta: 0.0776 | Regression_pt: 0.5302 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 5.1301 | physical_nopu: 6.0772 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6759 | Classification_binary: 4.1293 | Regression_cos_phi: 1.6670 | Regression_energy: 3.4535 | Regression_eta: 1.1342 | Regression_pt: 4.3538 | Regression_sin_phi: 1.9725 | LR: 4.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.3143s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0014 | max_forward: 0.0387 | max_loss: 0.0036 | max_backward: 0.0664 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 1.2817 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 4374.6 | uss_MB: 3102.3 | child_rss_MB: 11636.1 | allocated_MB: 609.1 | reserved_MB: 27788.0 | inactive_split_MB: 244.1 +rank2 [2026-10-27 16:18:42,007] INFO : Step 2000/100000 rank2 | Train Loss: 17.7204 | Train Loss Unweighted: 5.4242 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.9240 | Regression_cos_phi: 0.0505 | Regression_energy: 0.6758 | Regression_eta: 0.0665 | Regression_pt: 0.6366 | Regression_sin_phi: 0.0551 | Group Train Loss Unweighted (window mean): physical_pu: 4.2802 | physical_nopu: 5.3470 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4190 | Classification_binary: 4.0954 | Regression_cos_phi: 2.3661 | Regression_energy: 2.3540 | Regression_eta: 1.1100 | Regression_pt: 2.8462 | Regression_sin_phi: 1.9345 | LR: 4.00e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8064s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1422 | max_loss: 0.0036 | max_backward: 0.1717 | max_optimizer_step: 0.1501 | max_tensorboard_logging: 0.7287 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 4367.5 | uss_MB: 3102.8 | child_rss_MB: 11637.6 | allocated_MB: 607.0 | reserved_MB: 27788.0 | inactive_split_MB: 326.9 +rank2 [2026-10-27 16:22:10,507] INFO : Step 3000/100000 rank2 | Train Loss: 14.3102 | Train Loss Unweighted: 4.8875 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.8970 | Regression_cos_phi: 0.0491 | Regression_energy: 0.4277 | Regression_eta: 0.0716 | Regression_pt: 0.3769 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 4.0705 | physical_nopu: 5.1319 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4715 | Classification_binary: 4.0662 | Regression_cos_phi: 1.9590 | Regression_energy: 2.4489 | Regression_eta: 1.2078 | Regression_pt: 2.7376 | Regression_sin_phi: 2.2708 | LR: 3.99e-04 | DataLoad Time: 0.0015s | Model Forward Time: 2.0076s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0782 | max_loss: 0.0037 | max_backward: 0.0505 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 1.0271 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4295.2 | uss_MB: 3034.9 | child_rss_MB: 11625.5 | allocated_MB: 607.6 | reserved_MB: 28576.0 | inactive_split_MB: 261.0 +rank2 [2026-10-27 16:25:44,195] INFO : Step 4000/100000 rank2 | Train Loss: 12.6160 | Train Loss Unweighted: 3.9080 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.1145 | Regression_cos_phi: 0.0484 | Regression_energy: 0.3088 | Regression_eta: 0.1117 | Regression_pt: 0.2597 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.9908 | physical_nopu: 5.0219 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5146 | Classification_binary: 3.1344 | Regression_cos_phi: 1.7723 | Regression_energy: 1.6520 | Regression_eta: 1.5865 | Regression_pt: 1.8358 | Regression_sin_phi: 1.9581 | LR: 3.99e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.4273s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0032 | max_forward: 0.1365 | max_loss: 0.0036 | max_backward: 0.2496 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 1.0566 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 4292.4 | uss_MB: 3043.1 | child_rss_MB: 11491.5 | allocated_MB: 617.3 | reserved_MB: 28576.0 | inactive_split_MB: 1049.1 +rank2 [2026-10-27 16:29:29,026] INFO : Step 5000/100000 rank2 | Train Loss: 13.7309 | Train Loss Unweighted: 4.7575 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.8762 | Regression_cos_phi: 0.0508 | Regression_energy: 0.3661 | Regression_eta: 0.0922 | Regression_pt: 0.3105 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.9453 | physical_nopu: 4.9561 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3910 | Classification_binary: 3.8530 | Regression_cos_phi: 1.7385 | Regression_energy: 1.8970 | Regression_eta: 1.0841 | Regression_pt: 2.0956 | Regression_sin_phi: 2.0356 | LR: 3.98e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.0872s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.0008 | max_forward: 0.0371 | max_loss: 0.0036 | max_backward: 0.0568 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 2.1174 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4241.6 | uss_MB: 3036.2 | child_rss_MB: 11146.9 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 248.7 +rank2 [2026-10-27 16:33:03,739] INFO : Step 6000/100000 rank2 | Train Loss: 15.2316 | Train Loss Unweighted: 4.9080 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.7479 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5147 | Regression_eta: 0.0679 | Regression_pt: 0.4609 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.9059 | physical_nopu: 4.9043 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5044 | Classification_binary: 3.6324 | Regression_cos_phi: 1.5560 | Regression_energy: 3.5608 | Regression_eta: 0.7414 | Regression_pt: 4.3338 | Regression_sin_phi: 1.4789 | LR: 3.97e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.0044s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0421 | max_loss: 0.0036 | max_backward: 0.0620 | max_optimizer_step: 0.1489 | max_tensorboard_logging: 2.0592 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4216.9 | uss_MB: 3036.3 | child_rss_MB: 11044.6 | allocated_MB: 610.3 | reserved_MB: 28576.0 | inactive_split_MB: 254.0 +rank2 [2026-10-27 16:36:49,903] INFO : Step 7000/100000 rank2 | Train Loss: 14.7215 | Train Loss Unweighted: 4.8917 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8534 | Regression_cos_phi: 0.0503 | Regression_energy: 0.4508 | Regression_eta: 0.0791 | Regression_pt: 0.3921 | Regression_sin_phi: 0.0547 | Group Train Loss Unweighted (window mean): physical_pu: 3.8763 | physical_nopu: 4.8627 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3648 | Classification_binary: 3.9559 | Regression_cos_phi: 2.1992 | Regression_energy: 1.7292 | Regression_eta: 1.0928 | Regression_pt: 1.9502 | Regression_sin_phi: 2.2293 | LR: 3.96e-04 | DataLoad Time: 0.0016s | Model Forward Time: 4.5448s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0348 | max_loss: 0.0036 | max_backward: 0.0535 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 3.5409 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4194.7 | uss_MB: 3033.5 | child_rss_MB: 10983.9 | allocated_MB: 609.0 | reserved_MB: 28576.0 | inactive_split_MB: 216.2 +rank2 [2026-10-27 16:40:34,139] INFO : Step 8000/100000 rank2 | Train Loss: 14.2560 | Train Loss Unweighted: 4.8042 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.8045 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4333 | Regression_eta: 0.0601 | Regression_pt: 0.3883 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.8518 | physical_nopu: 4.8294 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5532 | Classification_binary: 3.8090 | Regression_cos_phi: 2.0697 | Regression_energy: 2.3444 | Regression_eta: 0.7402 | Regression_pt: 2.9798 | Regression_sin_phi: 2.1991 | LR: 3.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 4.8859s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1342 | max_loss: 0.0036 | max_backward: 0.0500 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 3.8645 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4176.9 | uss_MB: 3036.4 | child_rss_MB: 10934.3 | allocated_MB: 606.6 | reserved_MB: 28576.0 | inactive_split_MB: 255.4 +rank2 [2026-10-27 16:44:13,307] INFO : Step 9000/100000 rank2 | Train Loss: 15.9274 | Train Loss Unweighted: 4.9361 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.6437 | Regression_cos_phi: 0.0472 | Regression_energy: 0.5701 | Regression_eta: 0.0733 | Regression_pt: 0.5360 | Regression_sin_phi: 0.0496 | Group Train Loss Unweighted (window mean): physical_pu: 3.8376 | physical_nopu: 4.8057 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5093 | Classification_binary: 3.7212 | Regression_cos_phi: 1.7362 | Regression_energy: 4.1216 | Regression_eta: 1.6691 | Regression_pt: 5.5607 | Regression_sin_phi: 2.7189 | LR: 3.93e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.2106s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1320 | max_loss: 0.0037 | max_backward: 0.0555 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 2.0797 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 4157.5 | uss_MB: 3035.9 | child_rss_MB: 10877.2 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 321.4 +rank2 [2026-10-27 16:48:05,432] INFO : Step 10000/100000 rank2 | Train Loss: 13.0514 | Train Loss Unweighted: 4.5682 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.7678 | Regression_cos_phi: 0.0520 | Regression_energy: 0.3372 | Regression_eta: 0.0736 | Regression_pt: 0.2745 | Regression_sin_phi: 0.0521 | Group Train Loss Unweighted (window mean): physical_pu: 3.8237 | physical_nopu: 4.7797 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2726 | Classification_binary: 3.6371 | Regression_cos_phi: 1.5869 | Regression_energy: 1.7964 | Regression_eta: 1.3920 | Regression_pt: 2.0100 | Regression_sin_phi: 1.7392 | LR: 3.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.2565s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.0379 | max_loss: 0.0040 | max_backward: 0.0556 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 2.2516 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4149.4 | uss_MB: 3036.1 | child_rss_MB: 10872.4 | allocated_MB: 607.9 | reserved_MB: 28576.0 | inactive_split_MB: 258.7 +rank2 [2026-10-27 16:48:06,334] INFO : Running validation on rank2 +rank2 [2026-10-27 16:48:06,334] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 16:48:06,334] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 16:50:06,285] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:50:06,771] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 16:50:34,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 16:50:35,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 16:50:36,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 16:50:36,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 16:50:37,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 16:50:37,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 16:50:38,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 16:50:38,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 16:50:48,687] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-27 16:50:48,968] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:50:49,335] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 16:51:04,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 16:51:05,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 16:51:05,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 16:51:06,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 16:51:07,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 16:51:07,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 16:51:08,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 16:51:08,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 16:51:18,563] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 16:51:18,736] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 16:51:19,153] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 16:51:34,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 16:51:35,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 16:51:35,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 16:51:36,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 16:51:36,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 16:51:37,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 16:51:37,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 16:51:38,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 16:51:48,283] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 16:55:58,693] INFO : Step 11000/100000 rank2 | Train Loss: 14.4095 | Train Loss Unweighted: 4.7539 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.7591 | Regression_cos_phi: 0.0522 | Regression_energy: 0.4228 | Regression_eta: 0.0709 | Regression_pt: 0.3796 | Regression_sin_phi: 0.0518 | Group Train Loss Unweighted (window mean): physical_pu: 3.8077 | physical_nopu: 4.7444 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5838 | Classification_binary: 3.7190 | Regression_cos_phi: 1.6231 | Regression_energy: 2.0736 | Regression_eta: 0.9324 | Regression_pt: 2.5958 | Regression_sin_phi: 1.9292 | LR: 3.89e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.1995s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1127 | max_loss: 0.0074 | max_backward: 0.1410 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 1.9404 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6580.9 | uss_MB: 5462.7 | child_rss_MB: 10900.1 | allocated_MB: 610.6 | reserved_MB: 27422.0 | inactive_split_MB: 253.4 +rank2 [2026-10-27 16:59:46,314] INFO : Step 12000/100000 rank2 | Train Loss: 16.4725 | Train Loss Unweighted: 5.0863 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.5958 | Regression_cos_phi: 0.0410 | Regression_energy: 0.6842 | Regression_eta: 0.0610 | Regression_pt: 0.6484 | Regression_sin_phi: 0.0394 | Group Train Loss Unweighted (window mean): physical_pu: 3.8013 | physical_nopu: 4.7308 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5623 | Classification_binary: 3.5704 | Regression_cos_phi: 1.2639 | Regression_energy: 4.0142 | Regression_eta: 0.7130 | Regression_pt: 5.1634 | Regression_sin_phi: 1.6267 | LR: 3.87e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.5370s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1292 | max_loss: 0.0037 | max_backward: 0.1951 | max_optimizer_step: 0.2940 | max_tensorboard_logging: 2.1160 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6552.9 | uss_MB: 5462.9 | child_rss_MB: 10800.4 | allocated_MB: 609.5 | reserved_MB: 27422.0 | inactive_split_MB: 347.2 +rank2 [2026-10-27 17:03:40,557] INFO : Step 13000/100000 rank2 | Train Loss: 12.9993 | Train Loss Unweighted: 4.5752 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.7655 | Regression_cos_phi: 0.0483 | Regression_energy: 0.3375 | Regression_eta: 0.0674 | Regression_pt: 0.2918 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7898 | physical_nopu: 4.7063 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4075 | Classification_binary: 4.0381 | Regression_cos_phi: 1.8762 | Regression_energy: 1.6782 | Regression_eta: 0.6734 | Regression_pt: 1.8900 | Regression_sin_phi: 2.0627 | LR: 3.85e-04 | DataLoad Time: 0.0087s | Model Forward Time: 3.0970s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1327 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.9573 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5436.1 | uss_MB: 4383.7 | child_rss_MB: 10499.2 | allocated_MB: 607.2 | reserved_MB: 28584.0 | inactive_split_MB: 340.8 +rank2 [2026-10-27 17:07:23,686] INFO : Step 14000/100000 rank2 | Train Loss: 12.1035 | Train Loss Unweighted: 3.7686 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.0394 | Regression_cos_phi: 0.0492 | Regression_energy: 0.2767 | Regression_eta: 0.1064 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0483 | Group Train Loss Unweighted (window mean): physical_pu: 3.7803 | physical_nopu: 4.6878 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5035 | Classification_binary: 2.9878 | Regression_cos_phi: 1.8351 | Regression_energy: 1.4788 | Regression_eta: 1.5601 | Regression_pt: 1.6777 | Regression_sin_phi: 1.8593 | LR: 3.83e-04 | DataLoad Time: 0.0019s | Model Forward Time: 5.5486s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0034 | max_forward: 0.2002 | max_loss: 0.0036 | max_backward: 0.3668 | max_optimizer_step: 0.2700 | max_tensorboard_logging: 3.9098 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0824 | Memory: rss_MB: 5443.1 | uss_MB: 4393.2 | child_rss_MB: 10497.0 | allocated_MB: 618.5 | reserved_MB: 28584.0 | inactive_split_MB: 893.5 +rank2 [2026-10-27 17:11:07,917] INFO : Step 15000/100000 rank2 | Train Loss: 13.2483 | Train Loss Unweighted: 4.5172 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.6965 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3441 | Regression_eta: 0.0751 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7702 | physical_nopu: 4.6706 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3722 | Classification_binary: 3.6456 | Regression_cos_phi: 2.2190 | Regression_energy: 1.7789 | Regression_eta: 1.1384 | Regression_pt: 1.9901 | Regression_sin_phi: 1.7138 | LR: 3.80e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.3454s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0879 | max_loss: 0.0036 | max_backward: 0.0859 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 2.2540 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0190 | Memory: rss_MB: 5437.1 | uss_MB: 4385.0 | child_rss_MB: 10498.2 | allocated_MB: 605.9 | reserved_MB: 28584.0 | inactive_split_MB: 314.1 +rank2 [2026-10-27 17:14:55,829] INFO : Step 16000/100000 rank2 | Train Loss: 14.0060 | Train Loss Unweighted: 4.5724 | Train Loss Components: Classification: 0.0150 | Classification_binary: 3.5827 | Regression_cos_phi: 0.0469 | Regression_energy: 0.4276 | Regression_eta: 0.0760 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0517 | Group Train Loss Unweighted (window mean): physical_pu: 3.7654 | physical_nopu: 4.6500 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5007 | Classification_binary: 3.4556 | Regression_cos_phi: 1.7599 | Regression_energy: 2.9428 | Regression_eta: 1.1567 | Regression_pt: 3.5824 | Regression_sin_phi: 1.9918 | LR: 3.78e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.4621s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0984 | max_forward: 0.1749 | max_loss: 0.0039 | max_backward: 0.0976 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 2.1733 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5435.1 | uss_MB: 4384.3 | child_rss_MB: 10495.1 | allocated_MB: 607.6 | reserved_MB: 28978.0 | inactive_split_MB: 330.4 +rank2 [2026-10-27 17:18:33,642] INFO : Step 17000/100000 rank2 | Train Loss: 14.2557 | Train Loss Unweighted: 4.6663 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6292 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4540 | Regression_eta: 0.0715 | Regression_pt: 0.3998 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.7576 | physical_nopu: 4.6430 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3965 | Classification_binary: 3.7232 | Regression_cos_phi: 2.1611 | Regression_energy: 2.9992 | Regression_eta: 1.0418 | Regression_pt: 3.5288 | Regression_sin_phi: 2.0272 | LR: 3.75e-04 | DataLoad Time: 0.0034s | Model Forward Time: 3.9696s | Phase Times: max_pre_step_barrier: 0.5705 | max_batch_to_device: 0.0043 | max_forward: 0.0887 | max_loss: 0.0037 | max_backward: 0.1154 | max_optimizer_step: 0.2542 | max_tensorboard_logging: 2.1801 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5421.0 | uss_MB: 4384.2 | child_rss_MB: 10406.9 | allocated_MB: 609.7 | reserved_MB: 29500.0 | inactive_split_MB: 302.1 +rank2 [2026-10-27 17:22:18,273] INFO : Step 18000/100000 rank2 | Train Loss: 13.2896 | Train Loss Unweighted: 4.5062 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6504 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3607 | Regression_eta: 0.0732 | Regression_pt: 0.3062 | Regression_sin_phi: 0.0530 | Group Train Loss Unweighted (window mean): physical_pu: 3.7500 | physical_nopu: 4.6286 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3903 | Classification_binary: 3.6712 | Regression_cos_phi: 1.8026 | Regression_energy: 1.9846 | Regression_eta: 1.2367 | Regression_pt: 2.1568 | Regression_sin_phi: 2.2189 | LR: 3.72e-04 | DataLoad Time: 0.0020s | Model Forward Time: 10.0051s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0832 | max_loss: 0.0037 | max_backward: 0.0999 | max_optimizer_step: 0.1725 | max_tensorboard_logging: 8.8254 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 5416.7 | uss_MB: 4384.0 | child_rss_MB: 10363.7 | allocated_MB: 608.8 | reserved_MB: 29500.0 | inactive_split_MB: 196.1 +rank2 [2026-10-27 17:25:51,406] INFO : Step 19000/100000 rank2 | Train Loss: 15.2449 | Train Loss Unweighted: 4.8480 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5669 | Regression_cos_phi: 0.0399 | Regression_energy: 0.5792 | Regression_eta: 0.0602 | Regression_pt: 0.5415 | Regression_sin_phi: 0.0443 | Group Train Loss Unweighted (window mean): physical_pu: 3.7399 | physical_nopu: 4.6217 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4907 | Classification_binary: 3.6231 | Regression_cos_phi: 1.6444 | Regression_energy: 2.5279 | Regression_eta: 0.9351 | Regression_pt: 3.0370 | Regression_sin_phi: 1.8600 | LR: 3.69e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.1642s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0008 | max_forward: 0.0818 | max_loss: 0.0762 | max_backward: 0.1759 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 1.9822 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5410.7 | uss_MB: 4384.1 | child_rss_MB: 10330.8 | allocated_MB: 607.1 | reserved_MB: 29500.0 | inactive_split_MB: 236.2 +rank2 [2026-10-27 17:29:45,026] INFO : Step 20000/100000 rank2 | Train Loss: 13.7275 | Train Loss Unweighted: 4.6495 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.7120 | Regression_cos_phi: 0.0510 | Regression_energy: 0.4061 | Regression_eta: 0.0646 | Regression_pt: 0.3531 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.7387 | physical_nopu: 4.6038 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3676 | Classification_binary: 3.7814 | Regression_cos_phi: 1.8133 | Regression_energy: 1.6157 | Regression_eta: 0.8316 | Regression_pt: 1.8239 | Regression_sin_phi: 1.8281 | LR: 3.66e-04 | DataLoad Time: 0.0027s | Model Forward Time: 5.4931s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0007 | max_forward: 0.0851 | max_loss: 0.0039 | max_backward: 0.2016 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 4.2823 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5411.1 | uss_MB: 4383.8 | child_rss_MB: 10307.1 | allocated_MB: 609.5 | reserved_MB: 29500.0 | inactive_split_MB: 252.8 +rank2 [2026-10-27 17:29:45,969] INFO : Running validation on rank2 +rank2 [2026-10-27 17:29:45,969] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 17:31:42,604] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 17:31:43,255] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 17:32:09,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 17:32:10,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 17:32:10,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 17:32:11,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 17:32:11,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 17:32:12,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 17:32:12,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 17:32:13,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 17:32:23,496] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-27 17:32:23,636] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 17:32:24,023] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 17:32:38,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 17:32:39,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 17:32:39,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 17:32:40,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 17:32:40,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 17:32:41,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 17:32:41,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 17:32:42,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 17:32:52,448] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 17:32:53,432] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 17:32:53,822] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 17:33:08,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 17:33:09,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 17:33:09,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 17:33:10,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 17:33:10,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 17:33:11,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 17:33:11,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 17:33:12,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 17:33:22,375] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 17:37:33,418] INFO : Step 21000/100000 rank2 | Train Loss: 13.1647 | Train Loss Unweighted: 4.3967 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.5106 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3742 | Regression_eta: 0.0796 | Regression_pt: 0.3229 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.7328 | physical_nopu: 4.5933 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5590 | Classification_binary: 3.4345 | Regression_cos_phi: 1.6208 | Regression_energy: 1.9204 | Regression_eta: 1.0658 | Regression_pt: 2.2565 | Regression_sin_phi: 2.1445 | LR: 3.62e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.6673s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0483 | max_forward: 0.1678 | max_loss: 0.0037 | max_backward: 0.1430 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 2.3569 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5516.5 | uss_MB: 4216.2 | child_rss_MB: 10241.6 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 238.6 +rank2 [2026-10-27 17:41:10,019] INFO : Step 22000/100000 rank2 | Train Loss: 15.3802 | Train Loss Unweighted: 4.7914 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.5311 | Regression_cos_phi: 0.0475 | Regression_energy: 0.5593 | Regression_eta: 0.0730 | Regression_pt: 0.5219 | Regression_sin_phi: 0.0434 | Group Train Loss Unweighted (window mean): physical_pu: 3.7290 | physical_nopu: 4.5858 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4794 | Classification_binary: 3.5883 | Regression_cos_phi: 2.0252 | Regression_energy: 4.4078 | Regression_eta: 1.4651 | Regression_pt: 5.7582 | Regression_sin_phi: 2.1702 | LR: 3.59e-04 | DataLoad Time: 0.0009s | Model Forward Time: 3.1752s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0905 | max_loss: 0.0038 | max_backward: 0.1361 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 2.0169 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5517.5 | uss_MB: 4498.8 | child_rss_MB: 10247.2 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 321.0 +rank2 [2026-10-27 17:44:47,651] INFO : Step 23000/100000 rank2 | Train Loss: 12.9848 | Train Loss Unweighted: 4.5226 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.7377 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3326 | Regression_eta: 0.0739 | Regression_pt: 0.2617 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.7288 | physical_nopu: 4.5774 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2962 | Classification_binary: 3.5954 | Regression_cos_phi: 1.8709 | Regression_energy: 2.0465 | Regression_eta: 1.1619 | Regression_pt: 1.8991 | Regression_sin_phi: 2.2671 | LR: 3.55e-04 | DataLoad Time: 0.0070s | Model Forward Time: 3.1587s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0994 | max_loss: 0.0039 | max_backward: 0.1244 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.9838 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 5517.5 | uss_MB: 4498.0 | child_rss_MB: 10251.2 | allocated_MB: 608.2 | reserved_MB: 29422.0 | inactive_split_MB: 218.5 +rank2 [2026-10-27 17:48:44,022] INFO : Step 24000/100000 rank2 | Train Loss: 16.3896 | Train Loss Unweighted: 4.8949 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4510 | Regression_cos_phi: 0.0465 | Regression_energy: 0.6613 | Regression_eta: 0.0617 | Regression_pt: 0.6132 | Regression_sin_phi: 0.0453 | Group Train Loss Unweighted (window mean): physical_pu: 3.7170 | physical_nopu: 4.5767 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4495 | Classification_binary: 3.7560 | Regression_cos_phi: 1.6261 | Regression_energy: 2.3714 | Regression_eta: 1.1249 | Regression_pt: 2.8334 | Regression_sin_phi: 2.0065 | LR: 3.51e-04 | DataLoad Time: 0.0027s | Model Forward Time: 3.1024s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0398 | max_forward: 0.1314 | max_loss: 0.0037 | max_backward: 0.0994 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 1.8290 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5518.7 | uss_MB: 4498.4 | child_rss_MB: 10238.8 | allocated_MB: 606.3 | reserved_MB: 29422.0 | inactive_split_MB: 288.6 +rank2 [2026-10-27 17:52:29,098] INFO : Step 25000/100000 rank2 | Train Loss: 13.0062 | Train Loss Unweighted: 4.4048 | Train Loss Components: Classification: 0.0114 | Classification_binary: 3.6384 | Regression_cos_phi: 0.0552 | Regression_energy: 0.3153 | Regression_eta: 0.0680 | Regression_pt: 0.2574 | Regression_sin_phi: 0.0590 | Group Train Loss Unweighted (window mean): physical_pu: 3.7138 | physical_nopu: 4.5644 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3061 | Classification_binary: 3.5473 | Regression_cos_phi: 1.7539 | Regression_energy: 1.4017 | Regression_eta: 0.8236 | Regression_pt: 1.5358 | Regression_sin_phi: 1.7999 | LR: 3.47e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.5988s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0007 | max_forward: 0.1394 | max_loss: 0.0036 | max_backward: 0.2004 | max_optimizer_step: 0.1603 | max_tensorboard_logging: 1.3791 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5517.7 | uss_MB: 4498.3 | child_rss_MB: 10255.6 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 213.9 +rank2 [2026-10-27 17:56:10,619] INFO : Step 26000/100000 rank2 | Train Loss: 13.5469 | Train Loss Unweighted: 4.4523 | Train Loss Components: Classification: 0.0185 | Classification_binary: 3.5246 | Regression_cos_phi: 0.0498 | Regression_energy: 0.3935 | Regression_eta: 0.0732 | Regression_pt: 0.3461 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5624 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4794 | Classification_binary: 3.5123 | Regression_cos_phi: 1.8777 | Regression_energy: 2.6916 | Regression_eta: 1.0532 | Regression_pt: 3.1206 | Regression_sin_phi: 1.6172 | LR: 3.43e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.1211s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0838 | max_forward: 0.1679 | max_loss: 0.0039 | max_backward: 0.0991 | max_optimizer_step: 0.2674 | max_tensorboard_logging: 1.6325 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5515.7 | uss_MB: 4498.4 | child_rss_MB: 10248.9 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 201.9 +rank2 [2026-10-27 17:59:48,627] INFO : Step 27000/100000 rank2 | Train Loss: 17.1780 | Train Loss Unweighted: 5.0531 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.5146 | Regression_cos_phi: 0.0448 | Regression_energy: 0.6955 | Regression_eta: 0.0778 | Regression_pt: 0.6583 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5579 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5263 | Classification_binary: 3.5883 | Regression_cos_phi: 1.7899 | Regression_energy: 3.3757 | Regression_eta: 0.9840 | Regression_pt: 4.3820 | Regression_sin_phi: 2.0276 | LR: 3.39e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.4477s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1019 | max_loss: 0.0489 | max_backward: 0.1596 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5507.7 | uss_MB: 4499.5 | child_rss_MB: 10244.8 | allocated_MB: 611.7 | reserved_MB: 30248.0 | inactive_split_MB: 336.3 +rank2 [2026-10-27 18:03:26,753] INFO : Step 28000/100000 rank2 | Train Loss: 12.6721 | Train Loss Unweighted: 4.3436 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5550 | Regression_cos_phi: 0.0488 | Regression_energy: 0.3264 | Regression_eta: 0.0754 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7057 | physical_nopu: 4.5482 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3009 | Classification_binary: 3.2085 | Regression_cos_phi: 1.7527 | Regression_energy: 1.7550 | Regression_eta: 0.9490 | Regression_pt: 1.9655 | Regression_sin_phi: 1.7406 | LR: 3.35e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.0160s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0898 | max_loss: 0.0057 | max_backward: 0.1164 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.8021 | max_tensorboard_flush: 0.0040 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 5489.8 | uss_MB: 4496.9 | child_rss_MB: 10097.2 | allocated_MB: 606.6 | reserved_MB: 30248.0 | inactive_split_MB: 278.7 +rank2 [2026-10-27 18:07:25,425] INFO : Step 29000/100000 rank2 | Train Loss: 14.8403 | Train Loss Unweighted: 4.5753 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.4458 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4901 | Regression_eta: 0.0767 | Regression_pt: 0.4459 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.7068 | physical_nopu: 4.5439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5861 | Classification_binary: 3.2935 | Regression_cos_phi: 1.5809 | Regression_energy: 2.6227 | Regression_eta: 1.0758 | Regression_pt: 3.3805 | Regression_sin_phi: 1.7288 | LR: 3.30e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.3386s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1464 | max_loss: 0.0037 | max_backward: 0.0935 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 1.1535 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5489.8 | uss_MB: 4498.4 | child_rss_MB: 10094.9 | allocated_MB: 607.4 | reserved_MB: 30248.0 | inactive_split_MB: 313.3 +rank2 [2026-10-27 18:11:12,494] INFO : Step 30000/100000 rank2 | Train Loss: 13.2916 | Train Loss Unweighted: 4.4954 | Train Loss Components: Classification: 0.0126 | Classification_binary: 3.5989 | Regression_cos_phi: 0.0482 | Regression_energy: 0.3865 | Regression_eta: 0.0707 | Regression_pt: 0.3291 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7005 | physical_nopu: 4.5378 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3565 | Classification_binary: 3.4826 | Regression_cos_phi: 2.0900 | Regression_energy: 1.7592 | Regression_eta: 0.8463 | Regression_pt: 1.9446 | Regression_sin_phi: 1.7270 | LR: 3.26e-04 | DataLoad Time: 0.0015s | Model Forward Time: 2.7233s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0805 | max_loss: 0.0038 | max_backward: 0.0926 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 1.5279 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 5479.8 | uss_MB: 4498.3 | child_rss_MB: 10016.6 | allocated_MB: 608.2 | reserved_MB: 30760.0 | inactive_split_MB: 232.9 +rank2 [2026-10-27 18:11:13,431] INFO : Running validation on rank2 +rank2 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 18:13:07,587] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 18:13:08,046] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 18:13:34,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 18:13:35,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 18:13:35,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 18:13:36,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 18:13:36,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 18:13:37,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 18:13:37,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 18:13:38,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 18:13:48,179] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 18:13:48,509] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 18:13:49,355] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 18:14:04,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 18:14:05,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 18:14:06,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 18:14:06,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 18:14:07,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 18:14:07,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 18:14:08,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 18:14:08,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 18:14:18,687] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-27 18:14:18,747] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 18:14:19,219] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 18:14:34,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 18:14:35,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 18:14:35,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 18:14:36,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 18:14:36,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 18:14:37,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 18:14:37,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 18:14:38,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 18:14:48,127] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 18:19:00,303] INFO : Step 31000/100000 rank2 | Train Loss: 12.7750 | Train Loss Unweighted: 4.3889 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5406 | Regression_cos_phi: 0.0465 | Regression_energy: 0.3622 | Regression_eta: 0.0700 | Regression_pt: 0.3101 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6964 | physical_nopu: 4.5360 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4754 | Classification_binary: 3.2955 | Regression_cos_phi: 1.7972 | Regression_energy: 1.9487 | Regression_eta: 0.7828 | Regression_pt: 2.3104 | Regression_sin_phi: 1.6943 | LR: 3.21e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.3239s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.2346 | max_loss: 0.0037 | max_backward: 0.1255 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 1.8384 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1395 | Memory: rss_MB: 6773.5 | uss_MB: 5791.8 | child_rss_MB: 10011.8 | allocated_MB: 609.3 | reserved_MB: 29142.0 | inactive_split_MB: 278.5 +rank2 [2026-10-27 18:22:44,787] INFO : Step 32000/100000 rank2 | Train Loss: 14.3617 | Train Loss Unweighted: 4.6225 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4934 | Regression_cos_phi: 0.0448 | Regression_energy: 0.5043 | Regression_eta: 0.0718 | Regression_pt: 0.4507 | Regression_sin_phi: 0.0414 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5314 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4987 | Classification_binary: 3.5494 | Regression_cos_phi: 1.5302 | Regression_energy: 2.7004 | Regression_eta: 0.9788 | Regression_pt: 3.4569 | Regression_sin_phi: 0.9128 | LR: 3.16e-04 | DataLoad Time: 0.0102s | Model Forward Time: 2.5623s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1373 | max_loss: 0.0041 | max_backward: 0.2040 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 1.2065 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5556.1 | uss_MB: 4574.8 | child_rss_MB: 10008.9 | allocated_MB: 606.1 | reserved_MB: 29142.0 | inactive_split_MB: 511.2 +rank2 [2026-10-27 18:26:33,026] INFO : Step 33000/100000 rank2 | Train Loss: 12.0000 | Train Loss Unweighted: 3.7115 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9674 | Regression_cos_phi: 0.0474 | Regression_energy: 0.2863 | Regression_eta: 0.1056 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4974 | Classification_binary: 2.9099 | Regression_cos_phi: 1.7770 | Regression_energy: 1.6304 | Regression_eta: 1.5881 | Regression_pt: 1.8507 | Regression_sin_phi: 1.8423 | LR: 3.12e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.5986s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0030 | max_forward: 0.1619 | max_loss: 0.0119 | max_backward: 0.2845 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.1927 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5557.1 | uss_MB: 4556.3 | child_rss_MB: 10013.2 | allocated_MB: 623.7 | reserved_MB: 29572.0 | inactive_split_MB: 1441.5 +rank2 [2026-10-27 18:30:15,971] INFO : Step 34000/100000 rank2 | Train Loss: 15.0627 | Train Loss Unweighted: 4.6899 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4670 | Regression_cos_phi: 0.0454 | Regression_energy: 0.5438 | Regression_eta: 0.0626 | Regression_pt: 0.5082 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6889 | physical_nopu: 4.5215 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5485 | Classification_binary: 3.3695 | Regression_cos_phi: 1.4810 | Regression_energy: 2.7319 | Regression_eta: 0.7811 | Regression_pt: 3.4606 | Regression_sin_phi: 1.5534 | LR: 3.07e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.3752s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0007 | max_forward: 0.1346 | max_loss: 0.0037 | max_backward: 0.1938 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1506 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5552.6 | uss_MB: 4573.6 | child_rss_MB: 9984.9 | allocated_MB: 609.6 | reserved_MB: 29572.0 | inactive_split_MB: 230.4 +rank2 [2026-10-27 18:33:55,272] INFO : Step 35000/100000 rank2 | Train Loss: 12.2019 | Train Loss Unweighted: 4.3540 | Train Loss Components: Classification: 0.0089 | Classification_binary: 3.6426 | Regression_cos_phi: 0.0532 | Regression_energy: 0.2965 | Regression_eta: 0.0664 | Regression_pt: 0.2370 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6854 | physical_nopu: 4.5184 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2762 | Classification_binary: 3.5981 | Regression_cos_phi: 2.2563 | Regression_energy: 1.5317 | Regression_eta: 1.0652 | Regression_pt: 1.6730 | Regression_sin_phi: 1.9049 | LR: 3.02e-04 | DataLoad Time: 0.0011s | Model Forward Time: 2.6478s | Phase Times: max_pre_step_barrier: 0.0511 | max_batch_to_device: 0.0857 | max_forward: 0.1832 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 1.2656 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 5548.6 | uss_MB: 4574.4 | child_rss_MB: 9918.7 | allocated_MB: 608.2 | reserved_MB: 29572.0 | inactive_split_MB: 245.8 +rank2 [2026-10-27 18:37:33,851] INFO : Step 36000/100000 rank2 | Train Loss: 12.7130 | Train Loss Unweighted: 4.3877 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.5479 | Regression_cos_phi: 0.0433 | Regression_energy: 0.3521 | Regression_eta: 0.0636 | Regression_pt: 0.3167 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.6860 | physical_nopu: 4.5137 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5180 | Classification_binary: 3.4631 | Regression_cos_phi: 2.1241 | Regression_energy: 2.1780 | Regression_eta: 1.1856 | Regression_pt: 2.5368 | Regression_sin_phi: 2.3915 | LR: 2.97e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.5820s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0007 | max_forward: 0.1685 | max_loss: 0.0137 | max_backward: 0.0851 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.2906 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 5532.6 | uss_MB: 4487.8 | child_rss_MB: 9818.0 | allocated_MB: 609.0 | reserved_MB: 29572.0 | inactive_split_MB: 268.4 +rank2 [2026-10-27 18:41:12,499] INFO : Step 37000/100000 rank2 | Train Loss: 16.1273 | Train Loss Unweighted: 4.7522 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4049 | Regression_cos_phi: 0.0495 | Regression_energy: 0.5975 | Regression_eta: 0.0881 | Regression_pt: 0.5494 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6829 | physical_nopu: 4.5107 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4765 | Classification_binary: 3.3568 | Regression_cos_phi: 1.7579 | Regression_energy: 2.3295 | Regression_eta: 0.8661 | Regression_pt: 2.7411 | Regression_sin_phi: 1.9243 | LR: 2.91e-04 | DataLoad Time: 0.0027s | Model Forward Time: 2.3163s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0884 | max_loss: 0.0036 | max_backward: 0.0963 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 1.0635 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0656 | Memory: rss_MB: 5529.2 | uss_MB: 4574.6 | child_rss_MB: 9790.1 | allocated_MB: 605.6 | reserved_MB: 29572.0 | inactive_split_MB: 233.2 +rank2 [2026-10-27 18:44:44,282] INFO : Step 38000/100000 rank2 | Train Loss: 12.5142 | Train Loss Unweighted: 4.4218 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.6290 | Regression_cos_phi: 0.0475 | Regression_energy: 0.3376 | Regression_eta: 0.0660 | Regression_pt: 0.2827 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6875 | physical_nopu: 4.5136 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4678 | Classification_binary: 3.6504 | Regression_cos_phi: 1.8213 | Regression_energy: 1.7994 | Regression_eta: 0.9342 | Regression_pt: 1.8924 | Regression_sin_phi: 2.0553 | LR: 2.86e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.4325s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.0807 | max_loss: 0.0039 | max_backward: 0.1583 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.2386 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5519.2 | uss_MB: 4574.3 | child_rss_MB: 9713.9 | allocated_MB: 607.7 | reserved_MB: 29572.0 | inactive_split_MB: 203.1 +rank2 [2026-10-27 18:48:30,202] INFO : Step 39000/100000 rank2 | Train Loss: 13.9834 | Train Loss Unweighted: 4.4509 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4391 | Regression_cos_phi: 0.0477 | Regression_energy: 0.4363 | Regression_eta: 0.0792 | Regression_pt: 0.3839 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.6802 | physical_nopu: 4.5058 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5002 | Classification_binary: 3.4653 | Regression_cos_phi: 1.4522 | Regression_energy: 2.5886 | Regression_eta: 0.9324 | Regression_pt: 3.1820 | Regression_sin_phi: 2.2569 | LR: 2.81e-04 | DataLoad Time: 0.0027s | Model Forward Time: 3.8719s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.0950 | max_loss: 0.0039 | max_backward: 0.1102 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 2.6970 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5506.6 | uss_MB: 4574.6 | child_rss_MB: 9623.2 | allocated_MB: 611.4 | reserved_MB: 29682.0 | inactive_split_MB: 276.6 +rank2 [2026-10-27 18:52:07,824] INFO : Step 40000/100000 rank2 | Train Loss: 15.4467 | Train Loss Unweighted: 4.8241 | Train Loss Components: Classification: 0.0121 | Classification_binary: 3.5538 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5661 | Regression_eta: 0.0744 | Regression_pt: 0.5256 | Regression_sin_phi: 0.0462 | Group Train Loss Unweighted (window mean): physical_pu: 3.6754 | physical_nopu: 4.5045 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3166 | Classification_binary: 3.5199 | Regression_cos_phi: 1.5694 | Regression_energy: 1.6903 | Regression_eta: 0.9814 | Regression_pt: 1.9287 | Regression_sin_phi: 1.6873 | LR: 2.76e-04 | DataLoad Time: 0.0011s | Model Forward Time: 2.4749s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.1251 | max_loss: 0.0037 | max_backward: 0.0913 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2799 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5491.6 | uss_MB: 4574.5 | child_rss_MB: 9510.6 | allocated_MB: 607.6 | reserved_MB: 29682.0 | inactive_split_MB: 251.4 +rank2 [2026-10-27 18:52:08,683] INFO : Running validation on rank2 +rank2 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 18:54:11,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 18:54:12,296] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 18:54:37,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 18:54:38,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 18:54:38,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 18:54:39,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 18:54:39,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 18:54:40,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 18:54:40,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 18:54:41,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 18:54:51,233] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-27 18:54:51,318] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 18:54:51,717] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 18:55:07,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 18:55:07,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 18:55:08,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 18:55:08,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 18:55:09,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 18:55:09,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 18:55:10,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 18:55:10,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 18:55:20,698] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 18:55:21,167] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 18:55:21,566] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 18:55:36,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 18:55:36,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 18:55:37,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 18:55:38,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 18:55:38,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 18:55:39,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 18:55:39,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 18:55:40,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 18:55:50,096] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 19:00:08,492] INFO : Step 41000/100000 rank2 | Train Loss: 13.2101 | Train Loss Unweighted: 4.4466 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5620 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3777 | Regression_eta: 0.0720 | Regression_pt: 0.3238 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6774 | physical_nopu: 4.4925 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4553 | Classification_binary: 3.6873 | Regression_cos_phi: 1.7827 | Regression_energy: 2.2990 | Regression_eta: 1.2574 | Regression_pt: 2.6322 | Regression_sin_phi: 1.9994 | LR: 2.70e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.5583s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0904 | max_forward: 0.2227 | max_loss: 0.0038 | max_backward: 0.1239 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 2.1580 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6748.4 | uss_MB: 5831.9 | child_rss_MB: 9514.0 | allocated_MB: 608.3 | reserved_MB: 27230.0 | inactive_split_MB: 387.2 +rank2 [2026-10-27 19:03:45,853] INFO : Step 42000/100000 rank2 | Train Loss: 13.7061 | Train Loss Unweighted: 4.4760 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4108 | Regression_cos_phi: 0.0459 | Regression_energy: 0.4702 | Regression_eta: 0.0614 | Regression_pt: 0.4341 | Regression_sin_phi: 0.0374 | Group Train Loss Unweighted (window mean): physical_pu: 3.6711 | physical_nopu: 4.4967 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5565 | Classification_binary: 3.2882 | Regression_cos_phi: 1.1645 | Regression_energy: 2.2712 | Regression_eta: 0.8377 | Regression_pt: 2.6464 | Regression_sin_phi: 1.4706 | LR: 2.65e-04 | DataLoad Time: 0.0078s | Model Forward Time: 2.9851s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0008 | max_forward: 0.1920 | max_loss: 0.0038 | max_backward: 0.1427 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 1.6213 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6739.4 | uss_MB: 5628.8 | child_rss_MB: 9436.4 | allocated_MB: 609.4 | reserved_MB: 27632.0 | inactive_split_MB: 509.5 +rank2 [2026-10-27 19:07:23,506] INFO : Step 43000/100000 rank2 | Train Loss: 11.9489 | Train Loss Unweighted: 3.6938 | Train Loss Components: Classification: 0.0162 | Classification_binary: 2.9682 | Regression_cos_phi: 0.0472 | Regression_energy: 0.2764 | Regression_eta: 0.1089 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6702 | physical_nopu: 4.4944 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4843 | Classification_binary: 2.9574 | Regression_cos_phi: 1.7893 | Regression_energy: 1.5233 | Regression_eta: 1.6651 | Regression_pt: 1.6706 | Regression_sin_phi: 1.9619 | LR: 2.59e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.8328s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0032 | max_forward: 0.2195 | max_loss: 0.0114 | max_backward: 0.4513 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 2.0889 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0865 | Memory: rss_MB: 5560.5 | uss_MB: 3524.0 | child_rss_MB: 9348.2 | allocated_MB: 619.9 | reserved_MB: 27918.0 | inactive_split_MB: 1305.5 +rank2 [2026-10-27 19:11:14,852] INFO : Step 44000/100000 rank2 | Train Loss: 15.8790 | Train Loss Unweighted: 4.7893 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4610 | Regression_cos_phi: 0.0438 | Regression_energy: 0.5944 | Regression_eta: 0.0747 | Regression_pt: 0.5488 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6681 | physical_nopu: 4.4856 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4585 | Classification_binary: 3.5510 | Regression_cos_phi: 1.3010 | Regression_energy: 2.2444 | Regression_eta: 0.8628 | Regression_pt: 2.8527 | Regression_sin_phi: 1.4314 | LR: 2.54e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.1651s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0980 | max_loss: 0.0531 | max_backward: 0.1371 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.9403 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5553.5 | uss_MB: 4666.8 | child_rss_MB: 9287.7 | allocated_MB: 606.4 | reserved_MB: 31154.0 | inactive_split_MB: 295.0 +rank2 [2026-10-27 19:15:01,382] INFO : Step 45000/100000 rank2 | Train Loss: 13.8786 | Train Loss Unweighted: 4.5241 | Train Loss Components: Classification: 0.0100 | Classification_binary: 3.5822 | Regression_cos_phi: 0.0536 | Regression_energy: 0.4068 | Regression_eta: 0.0847 | Regression_pt: 0.3344 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 3.6667 | physical_nopu: 4.4836 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2385 | Classification_binary: 3.5786 | Regression_cos_phi: 2.1779 | Regression_energy: 1.6048 | Regression_eta: 1.1411 | Regression_pt: 1.4350 | Regression_sin_phi: 2.0979 | LR: 2.48e-04 | DataLoad Time: 0.0017s | Model Forward Time: 4.6040s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0009 | max_forward: 0.0951 | max_loss: 0.0815 | max_backward: 0.1837 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 3.3466 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 5551.4 | uss_MB: 4666.0 | child_rss_MB: 9284.1 | allocated_MB: 608.8 | reserved_MB: 31154.0 | inactive_split_MB: 289.8 +rank2 [2026-10-27 19:18:42,715] INFO : Step 46000/100000 rank2 | Train Loss: 13.1587 | Train Loss Unweighted: 4.3779 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.4817 | Regression_cos_phi: 0.0459 | Regression_energy: 0.3805 | Regression_eta: 0.0668 | Regression_pt: 0.3375 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.6620 | physical_nopu: 4.4831 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5501 | Classification_binary: 3.4376 | Regression_cos_phi: 1.5773 | Regression_energy: 2.0059 | Regression_eta: 1.2382 | Regression_pt: 2.2025 | Regression_sin_phi: 1.6764 | LR: 2.43e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.3332s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0924 | max_forward: 0.1910 | max_loss: 0.0039 | max_backward: 0.1028 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 2.0151 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5545.4 | uss_MB: 4666.2 | child_rss_MB: 9248.1 | allocated_MB: 609.2 | reserved_MB: 31154.0 | inactive_split_MB: 277.4 +rank2 [2026-10-27 19:22:31,522] INFO : Step 47000/100000 rank2 | Train Loss: 15.2118 | Train Loss Unweighted: 4.6831 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3624 | Regression_cos_phi: 0.0409 | Regression_energy: 0.5987 | Regression_eta: 0.0659 | Regression_pt: 0.5598 | Regression_sin_phi: 0.0405 | Group Train Loss Unweighted (window mean): physical_pu: 3.6609 | physical_nopu: 4.4795 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4628 | Classification_binary: 3.4870 | Regression_cos_phi: 1.3611 | Regression_energy: 3.2889 | Regression_eta: 0.8236 | Regression_pt: 4.1062 | Regression_sin_phi: 1.4053 | LR: 2.37e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.2742s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0777 | max_forward: 0.1864 | max_loss: 0.0077 | max_backward: 0.1105 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 1.8392 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5533.2 | uss_MB: 4665.8 | child_rss_MB: 9129.7 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 305.0 +rank2 [2026-10-27 19:26:21,057] INFO : Step 48000/100000 rank2 | Train Loss: 12.0455 | Train Loss Unweighted: 4.3014 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.6150 | Regression_cos_phi: 0.0526 | Regression_energy: 0.2805 | Regression_eta: 0.0652 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6617 | physical_nopu: 4.4726 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2822 | Classification_binary: 3.5260 | Regression_cos_phi: 1.7753 | Regression_energy: 1.2847 | Regression_eta: 0.7865 | Regression_pt: 1.4894 | Regression_sin_phi: 2.1075 | LR: 2.31e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.3650s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0531 | max_forward: 0.1607 | max_loss: 0.0044 | max_backward: 0.0997 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 2.0345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5534.2 | uss_MB: 4664.3 | child_rss_MB: 9129.2 | allocated_MB: 606.8 | reserved_MB: 31154.0 | inactive_split_MB: 262.3 +rank2 [2026-10-27 19:30:06,213] INFO : Step 49000/100000 rank2 | Train Loss: 13.3973 | Train Loss Unweighted: 4.4035 | Train Loss Components: Classification: 0.0168 | Classification_binary: 3.5033 | Regression_cos_phi: 0.0506 | Regression_energy: 0.3797 | Regression_eta: 0.0752 | Regression_pt: 0.3299 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6566 | physical_nopu: 4.4735 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4784 | Classification_binary: 3.4435 | Regression_cos_phi: 1.7033 | Regression_energy: 2.4156 | Regression_eta: 1.9900 | Regression_pt: 2.7003 | Regression_sin_phi: 2.0518 | LR: 2.26e-04 | DataLoad Time: 0.0018s | Model Forward Time: 4.5489s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0220 | max_forward: 0.1075 | max_loss: 0.0037 | max_backward: 0.1020 | max_optimizer_step: 0.2722 | max_tensorboard_logging: 3.2606 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5532.9 | uss_MB: 4389.0 | child_rss_MB: 9133.5 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 312.3 +rank2 [2026-10-27 19:33:47,826] INFO : Step 50000/100000 rank2 | Train Loss: 14.2027 | Train Loss Unweighted: 4.5299 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4272 | Regression_cos_phi: 0.0432 | Regression_energy: 0.4877 | Regression_eta: 0.0670 | Regression_pt: 0.4429 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4710 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5086 | Classification_binary: 3.4759 | Regression_cos_phi: 1.2076 | Regression_energy: 2.1662 | Regression_eta: 0.7559 | Regression_pt: 2.7863 | Regression_sin_phi: 2.0217 | LR: 2.20e-04 | DataLoad Time: 0.0013s | Model Forward Time: 3.0511s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0964 | max_loss: 0.0590 | max_backward: 0.1520 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.8031 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5519.9 | uss_MB: 4665.8 | child_rss_MB: 9015.1 | allocated_MB: 607.2 | reserved_MB: 31154.0 | inactive_split_MB: 255.6 +rank2 [2026-10-27 19:33:48,728] INFO : Running validation on rank2 +rank2 [2026-10-27 19:33:48,728] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 19:33:48,728] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 19:35:46,023] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 19:35:46,495] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 19:36:12,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 19:36:12,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 19:36:13,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 19:36:13,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 19:36:14,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 19:36:14,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 19:36:15,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 19:36:15,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 19:36:25,959] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-27 19:36:25,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 19:36:26,397] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 19:36:41,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 19:36:42,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 19:36:42,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 19:36:43,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 19:36:43,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 19:36:44,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 19:36:44,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 19:36:45,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 19:36:55,148] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 19:36:55,547] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 19:36:55,960] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 19:37:10,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 19:37:11,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 19:37:11,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 19:37:12,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 19:37:12,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 19:37:13,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 19:37:13,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 19:37:14,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 19:37:24,207] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 19:41:35,792] INFO : Step 51000/100000 rank2 | Train Loss: 13.0997 | Train Loss Unweighted: 4.3733 | Train Loss Components: Classification: 0.0129 | Classification_binary: 3.5282 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3598 | Regression_eta: 0.0785 | Regression_pt: 0.2920 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6545 | physical_nopu: 4.4631 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3723 | Classification_binary: 3.2916 | Regression_cos_phi: 1.3802 | Regression_energy: 1.6631 | Regression_eta: 0.8439 | Regression_pt: 2.0321 | Regression_sin_phi: 1.6510 | LR: 2.14e-04 | DataLoad Time: 0.0114s | Model Forward Time: 3.1040s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1313 | max_loss: 0.0271 | max_backward: 0.1689 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.7590 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6858.3 | uss_MB: 6017.9 | child_rss_MB: 8958.7 | allocated_MB: 607.6 | reserved_MB: 29536.0 | inactive_split_MB: 388.7 +rank2 [2026-10-27 19:45:15,993] INFO : Step 52000/100000 rank2 | Train Loss: 11.7496 | Train Loss Unweighted: 3.6574 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9475 | Regression_cos_phi: 0.0469 | Regression_energy: 0.2717 | Regression_eta: 0.1056 | Regression_pt: 0.2231 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6567 | physical_nopu: 4.4685 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5085 | Classification_binary: 2.9191 | Regression_cos_phi: 1.7488 | Regression_energy: 1.5019 | Regression_eta: 1.7078 | Regression_pt: 1.6528 | Regression_sin_phi: 1.9609 | LR: 2.09e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.6141s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.1059 | max_forward: 0.2956 | max_loss: 0.0038 | max_backward: 0.4187 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 1.8177 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6858.3 | uss_MB: 6026.1 | child_rss_MB: 8942.2 | allocated_MB: 619.9 | reserved_MB: 29990.0 | inactive_split_MB: 1078.6 +rank2 [2026-10-27 19:48:51,528] INFO : Step 53000/100000 rank2 | Train Loss: 12.5294 | Train Loss Unweighted: 4.3083 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5302 | Regression_cos_phi: 0.0494 | Regression_energy: 0.3242 | Regression_eta: 0.0722 | Regression_pt: 0.2705 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6530 | physical_nopu: 4.4629 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4185 | Classification_binary: 3.5935 | Regression_cos_phi: 2.1545 | Regression_energy: 2.0118 | Regression_eta: 1.2892 | Regression_pt: 2.2803 | Regression_sin_phi: 2.8589 | LR: 2.03e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.5888s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.1740 | max_loss: 0.0036 | max_backward: 0.1135 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 1.3124 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 5606.5 | uss_MB: 4766.5 | child_rss_MB: 8943.1 | allocated_MB: 608.5 | reserved_MB: 29990.0 | inactive_split_MB: 246.4 +rank2 [2026-10-27 19:52:28,508] INFO : Step 54000/100000 rank2 | Train Loss: 13.6142 | Train Loss Unweighted: 4.3355 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.3742 | Regression_cos_phi: 0.0499 | Regression_energy: 0.4083 | Regression_eta: 0.0736 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4652 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4720 | Classification_binary: 3.4337 | Regression_cos_phi: 2.0958 | Regression_energy: 2.0818 | Regression_eta: 0.8978 | Regression_pt: 2.3168 | Regression_sin_phi: 1.8901 | LR: 1.97e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.4442s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1029 | max_loss: 0.0156 | max_backward: 0.1263 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2459 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 5600.5 | uss_MB: 4766.6 | child_rss_MB: 8886.2 | allocated_MB: 608.0 | reserved_MB: 29990.0 | inactive_split_MB: 249.1 +rank2 [2026-10-27 19:56:02,481] INFO : Step 55000/100000 rank2 | Train Loss: 14.4463 | Train Loss Unweighted: 4.6356 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5250 | Regression_cos_phi: 0.0497 | Regression_energy: 0.4959 | Regression_eta: 0.0660 | Regression_pt: 0.4403 | Regression_sin_phi: 0.0455 | Group Train Loss Unweighted (window mean): physical_pu: 3.6517 | physical_nopu: 4.4554 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5329 | Classification_binary: 3.4073 | Regression_cos_phi: 1.6994 | Regression_energy: 2.4328 | Regression_eta: 0.8980 | Regression_pt: 2.8544 | Regression_sin_phi: 1.6915 | LR: 1.92e-04 | DataLoad Time: 0.0027s | Model Forward Time: 2.9798s | Phase Times: max_pre_step_barrier: 0.0108 | max_batch_to_device: 0.0008 | max_forward: 0.0923 | max_loss: 0.0219 | max_backward: 0.1306 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 1.7796 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5599.8 | uss_MB: 4765.5 | child_rss_MB: 8883.4 | allocated_MB: 610.0 | reserved_MB: 29990.0 | inactive_split_MB: 274.0 +rank2 [2026-10-27 19:59:40,797] INFO : Step 56000/100000 rank2 | Train Loss: 12.4091 | Train Loss Unweighted: 4.2572 | Train Loss Components: Classification: 0.0135 | Classification_binary: 3.4523 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3421 | Regression_eta: 0.0664 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6493 | physical_nopu: 4.4547 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3804 | Classification_binary: 3.3655 | Regression_cos_phi: 1.5538 | Regression_energy: 1.5490 | Regression_eta: 0.7298 | Regression_pt: 1.7543 | Regression_sin_phi: 1.8729 | LR: 1.86e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.0681s | Phase Times: max_pre_step_barrier: 0.3290 | max_batch_to_device: 0.0902 | max_forward: 0.1771 | max_loss: 0.0038 | max_backward: 0.1058 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 1.4273 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5599.8 | uss_MB: 4766.9 | child_rss_MB: 8880.9 | allocated_MB: 608.1 | reserved_MB: 29990.0 | inactive_split_MB: 253.9 +rank2 [2026-10-27 20:03:29,496] INFO : Step 57000/100000 rank2 | Train Loss: 14.9087 | Train Loss Unweighted: 4.5957 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4371 | Regression_cos_phi: 0.0471 | Regression_energy: 0.5028 | Regression_eta: 0.0770 | Regression_pt: 0.4658 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6475 | physical_nopu: 4.4509 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5417 | Classification_binary: 3.2798 | Regression_cos_phi: 1.6505 | Regression_energy: 3.3116 | Regression_eta: 1.0314 | Regression_pt: 4.1757 | Regression_sin_phi: 1.6999 | LR: 1.81e-04 | DataLoad Time: 0.0029s | Model Forward Time: 2.4954s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1036 | max_loss: 0.0876 | max_backward: 0.2130 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 1.1951 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5597.8 | uss_MB: 4497.3 | child_rss_MB: 8866.5 | allocated_MB: 608.1 | reserved_MB: 30358.0 | inactive_split_MB: 283.9 +rank2 [2026-10-27 20:07:08,735] INFO : Step 58000/100000 rank2 | Train Loss: 12.4468 | Train Loss Unweighted: 4.2826 | Train Loss Components: Classification: 0.0105 | Classification_binary: 3.5299 | Regression_cos_phi: 0.0545 | Regression_energy: 0.3106 | Regression_eta: 0.0620 | Regression_pt: 0.2643 | Regression_sin_phi: 0.0509 | Group Train Loss Unweighted (window mean): physical_pu: 3.6440 | physical_nopu: 4.4492 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4010 | Classification_binary: 3.4030 | Regression_cos_phi: 1.8815 | Regression_energy: 2.3705 | Regression_eta: 1.0529 | Regression_pt: 2.7957 | Regression_sin_phi: 2.0054 | LR: 1.75e-04 | DataLoad Time: 0.0027s | Model Forward Time: 2.5424s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1868 | max_loss: 0.0037 | max_backward: 0.1111 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2860 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5587.6 | uss_MB: 3611.2 | child_rss_MB: 8803.1 | allocated_MB: 608.2 | reserved_MB: 30358.0 | inactive_split_MB: 337.0 +rank2 [2026-10-27 20:10:52,442] INFO : Step 59000/100000 rank2 | Train Loss: 13.1870 | Train Loss Unweighted: 4.3659 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4760 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3771 | Regression_eta: 0.0773 | Regression_pt: 0.3247 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4440 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4442 | Classification_binary: 3.4375 | Regression_cos_phi: 1.8803 | Regression_energy: 1.9390 | Regression_eta: 0.9428 | Regression_pt: 2.2083 | Regression_sin_phi: 1.8896 | LR: 1.70e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.4660s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1005 | max_loss: 0.0036 | max_backward: 0.1062 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 1.2390 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 5582.6 | uss_MB: 4766.7 | child_rss_MB: 8755.9 | allocated_MB: 610.1 | reserved_MB: 31234.0 | inactive_split_MB: 273.9 +rank2 [2026-10-27 20:14:34,818] INFO : Step 60000/100000 rank2 | Train Loss: 15.4000 | Train Loss Unweighted: 4.6905 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.3905 | Regression_cos_phi: 0.0456 | Regression_energy: 0.5805 | Regression_eta: 0.0703 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6406 | physical_nopu: 4.4523 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5443 | Classification_binary: 3.5785 | Regression_cos_phi: 1.5307 | Regression_energy: 4.0883 | Regression_eta: 1.3079 | Regression_pt: 5.2530 | Regression_sin_phi: 1.4848 | LR: 1.64e-04 | DataLoad Time: 0.0014s | Model Forward Time: 2.5451s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0757 | max_forward: 0.1624 | max_loss: 0.0040 | max_backward: 0.1315 | max_optimizer_step: 0.2155 | max_tensorboard_logging: 1.2531 | max_tensorboard_flush: 0.0020 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5582.8 | uss_MB: 4766.6 | child_rss_MB: 8760.5 | allocated_MB: 611.6 | reserved_MB: 31234.0 | inactive_split_MB: 318.4 +rank2 [2026-10-27 20:14:35,735] INFO : Running validation on rank2 +rank2 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 20:16:30,817] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 20:16:31,261] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 20:16:55,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 20:16:56,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 20:16:56,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 20:16:57,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 20:16:57,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 20:16:58,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 20:16:58,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 20:16:59,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 20:17:09,155] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 20:17:09,354] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 20:17:09,707] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 20:17:25,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 20:17:25,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 20:17:26,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 20:17:26,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 20:17:27,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 20:17:27,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 20:17:28,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 20:17:28,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 20:17:38,911] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 20:17:39,021] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 20:17:39,450] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 20:17:54,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 20:17:55,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 20:17:55,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 20:17:56,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 20:17:56,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 20:17:57,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 20:17:57,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 20:17:58,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 20:18:08,144] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 20:22:28,768] INFO : Step 61000/100000 rank2 | Train Loss: 12.1474 | Train Loss Unweighted: 4.3289 | Train Loss Components: Classification: 0.0110 | Classification_binary: 3.6236 | Regression_cos_phi: 0.0523 | Regression_energy: 0.2909 | Regression_eta: 0.0654 | Regression_pt: 0.2368 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.6358 | physical_nopu: 4.4439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3133 | Classification_binary: 3.4942 | Regression_cos_phi: 2.0038 | Regression_energy: 1.7310 | Regression_eta: 0.7737 | Regression_pt: 1.8666 | Regression_sin_phi: 1.7436 | LR: 1.59e-04 | DataLoad Time: 0.0081s | Model Forward Time: 3.4724s | Phase Times: max_pre_step_barrier: 0.0115 | max_batch_to_device: 0.0009 | max_forward: 0.2092 | max_loss: 0.0037 | max_backward: 0.1501 | max_optimizer_step: 0.2965 | max_tensorboard_logging: 1.8725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6817.2 | uss_MB: 6009.9 | child_rss_MB: 8755.0 | allocated_MB: 607.8 | reserved_MB: 28822.0 | inactive_split_MB: 222.9 +rank2 [2026-10-27 20:26:14,158] INFO : Step 62000/100000 rank2 | Train Loss: 11.7977 | Train Loss Unweighted: 3.6294 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9084 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2778 | Regression_eta: 0.1039 | Regression_pt: 0.2286 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6417 | physical_nopu: 4.4459 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4744 | Classification_binary: 2.8820 | Regression_cos_phi: 1.7343 | Regression_energy: 1.5103 | Regression_eta: 1.5977 | Regression_pt: 1.6560 | Regression_sin_phi: 1.9817 | LR: 1.54e-04 | DataLoad Time: 0.0026s | Model Forward Time: 2.7423s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0033 | max_forward: 0.2219 | max_loss: 0.0037 | max_backward: 0.2965 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1292 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1278 | Memory: rss_MB: 6812.4 | uss_MB: 6017.7 | child_rss_MB: 8760.3 | allocated_MB: 618.8 | reserved_MB: 28822.0 | inactive_split_MB: 888.0 +rank2 [2026-10-27 20:30:05,877] INFO : Step 63000/100000 rank2 | Train Loss: 12.5071 | Train Loss Unweighted: 4.3613 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5997 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3200 | Regression_eta: 0.0719 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6399 | physical_nopu: 4.4447 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4205 | Classification_binary: 3.6095 | Regression_cos_phi: 1.8758 | Regression_energy: 1.7084 | Regression_eta: 1.0752 | Regression_pt: 1.7014 | Regression_sin_phi: 2.1231 | LR: 1.49e-04 | DataLoad Time: 0.0021s | Model Forward Time: 11.0660s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1169 | max_loss: 0.0157 | max_backward: 0.2882 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 9.0944 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6806.2 | uss_MB: 4694.0 | child_rss_MB: 8747.4 | allocated_MB: 610.4 | reserved_MB: 28822.0 | inactive_split_MB: 261.6 +rank2 [2026-10-27 20:33:53,443] INFO : Step 64000/100000 rank2 | Train Loss: 14.0388 | Train Loss Unweighted: 4.5035 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.4200 | Regression_cos_phi: 0.0430 | Regression_energy: 0.4752 | Regression_eta: 0.0663 | Regression_pt: 0.4376 | Regression_sin_phi: 0.0442 | Group Train Loss Unweighted (window mean): physical_pu: 3.6357 | physical_nopu: 4.4395 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6965 | Classification_binary: 3.3879 | Regression_cos_phi: 1.7400 | Regression_energy: 2.3980 | Regression_eta: 0.8720 | Regression_pt: 2.9797 | Regression_sin_phi: 1.7378 | LR: 1.43e-04 | DataLoad Time: 0.0023s | Model Forward Time: 6.4867s | Phase Times: max_pre_step_barrier: 0.1731 | max_batch_to_device: 0.0118 | max_forward: 0.1219 | max_loss: 0.0038 | max_backward: 0.1121 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 5.0441 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5600.0 | uss_MB: 4820.3 | child_rss_MB: 8744.3 | allocated_MB: 611.1 | reserved_MB: 29052.0 | inactive_split_MB: 372.9 +rank2 [2026-10-27 20:37:48,516] INFO : Step 65000/100000 rank2 | Train Loss: 16.3374 | Train Loss Unweighted: 4.8919 | Train Loss Components: Classification: 0.0142 | Classification_binary: 3.4871 | Regression_cos_phi: 0.0500 | Regression_energy: 0.6360 | Regression_eta: 0.0628 | Regression_pt: 0.5943 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4335 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4657 | Classification_binary: 3.5427 | Regression_cos_phi: 2.1617 | Regression_energy: 3.8748 | Regression_eta: 0.8641 | Regression_pt: 4.9883 | Regression_sin_phi: 2.0562 | LR: 1.38e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.4886s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.2167 | max_loss: 0.0058 | max_backward: 0.2407 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 1.9060 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5594.0 | uss_MB: 4820.8 | child_rss_MB: 8750.8 | allocated_MB: 609.7 | reserved_MB: 29052.0 | inactive_split_MB: 236.9 +rank2 [2026-10-27 20:41:37,390] INFO : Step 66000/100000 rank2 | Train Loss: 12.8848 | Train Loss Unweighted: 4.2935 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4715 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3451 | Regression_eta: 0.0693 | Regression_pt: 0.2907 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.6349 | physical_nopu: 4.4348 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4282 | Classification_binary: 3.6837 | Regression_cos_phi: 2.1348 | Regression_energy: 1.6618 | Regression_eta: 1.0419 | Regression_pt: 1.8734 | Regression_sin_phi: 2.2176 | LR: 1.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.1923s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0854 | max_loss: 0.1083 | max_backward: 0.2178 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.9218 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5582.1 | uss_MB: 4820.7 | child_rss_MB: 8750.8 | allocated_MB: 611.4 | reserved_MB: 29052.0 | inactive_split_MB: 254.2 +rank2 [2026-10-27 20:45:24,537] INFO : Step 67000/100000 rank2 | Train Loss: 14.2125 | Train Loss Unweighted: 4.5026 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.4493 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4566 | Regression_eta: 0.0671 | Regression_pt: 0.4150 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6352 | physical_nopu: 4.4350 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5098 | Classification_binary: 3.3800 | Regression_cos_phi: 2.0273 | Regression_energy: 2.9000 | Regression_eta: 1.2973 | Regression_pt: 3.2701 | Regression_sin_phi: 1.8589 | LR: 1.28e-04 | DataLoad Time: 0.0028s | Model Forward Time: 3.0117s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0901 | max_loss: 0.0487 | max_backward: 0.1449 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 1.8090 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5575.1 | uss_MB: 4820.8 | child_rss_MB: 8750.2 | allocated_MB: 607.1 | reserved_MB: 29260.0 | inactive_split_MB: 246.6 +rank2 [2026-10-27 20:49:05,126] INFO : Step 68000/100000 rank2 | Train Loss: 12.0487 | Train Loss Unweighted: 4.2633 | Train Loss Components: Classification: 0.0107 | Classification_binary: 3.5135 | Regression_cos_phi: 0.0466 | Regression_energy: 0.3165 | Regression_eta: 0.0645 | Regression_pt: 0.2647 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4387 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3275 | Classification_binary: 3.5367 | Regression_cos_phi: 2.1765 | Regression_energy: 1.7914 | Regression_eta: 0.9984 | Regression_pt: 2.1399 | Regression_sin_phi: 1.9174 | LR: 1.24e-04 | DataLoad Time: 0.0026s | Model Forward Time: 2.8889s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0010 | max_forward: 0.0879 | max_loss: 0.0120 | max_backward: 0.1069 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.7176 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5570.9 | uss_MB: 4820.5 | child_rss_MB: 8753.3 | allocated_MB: 609.5 | reserved_MB: 29260.0 | inactive_split_MB: 225.9 +rank2 [2026-10-27 20:52:47,283] INFO : Step 69000/100000 rank2 | Train Loss: 13.1377 | Train Loss Unweighted: 4.3133 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4496 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3632 | Regression_eta: 0.0778 | Regression_pt: 0.3079 | Regression_sin_phi: 0.0541 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4321 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4917 | Classification_binary: 3.4527 | Regression_cos_phi: 2.0116 | Regression_energy: 2.2900 | Regression_eta: 1.9321 | Regression_pt: 2.5792 | Regression_sin_phi: 2.5820 | LR: 1.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 4.9124s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.1207 | max_loss: 0.0509 | max_backward: 0.2012 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 3.5532 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 5571.9 | uss_MB: 4820.8 | child_rss_MB: 8755.8 | allocated_MB: 608.0 | reserved_MB: 29260.0 | inactive_split_MB: 256.4 +rank2 [2026-10-27 20:56:27,513] INFO : Step 70000/100000 rank2 | Train Loss: 15.8477 | Train Loss Unweighted: 4.7317 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3873 | Regression_cos_phi: 0.0498 | Regression_energy: 0.6015 | Regression_eta: 0.0629 | Regression_pt: 0.5683 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6315 | physical_nopu: 4.4280 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4365 | Classification_binary: 3.6426 | Regression_cos_phi: 1.8099 | Regression_energy: 3.0255 | Regression_eta: 0.8317 | Regression_pt: 3.5783 | Regression_sin_phi: 2.3516 | LR: 1.14e-04 | DataLoad Time: 0.0047s | Model Forward Time: 2.9390s | Phase Times: max_pre_step_barrier: 0.0198 | max_batch_to_device: 0.0007 | max_forward: 0.1178 | max_loss: 0.0036 | max_backward: 0.1081 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.7610 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5571.5 | uss_MB: 4819.7 | child_rss_MB: 8751.0 | allocated_MB: 610.3 | reserved_MB: 30150.0 | inactive_split_MB: 260.3 +rank2 [2026-10-27 20:56:28,457] INFO : Running validation on rank2 +rank2 [2026-10-27 20:56:28,457] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 20:56:28,457] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 20:58:25,954] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 20:58:26,360] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 20:58:51,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 20:58:52,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 20:58:53,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 20:58:53,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 20:58:54,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 20:58:54,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 20:58:55,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 20:58:55,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 20:59:05,631] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-27 20:59:05,946] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 20:59:06,379] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 20:59:21,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 20:59:22,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 20:59:22,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 20:59:23,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 20:59:23,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 20:59:24,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 20:59:24,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 20:59:25,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 20:59:35,256] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 20:59:35,862] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 20:59:36,279] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 20:59:51,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 20:59:51,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 20:59:52,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 20:59:52,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 20:59:53,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 20:59:53,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 20:59:54,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 20:59:54,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 21:00:04,551] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 21:04:25,115] INFO : Step 71000/100000 rank2 | Train Loss: 12.0110 | Train Loss Unweighted: 3.6539 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9046 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2881 | Regression_eta: 0.1063 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6306 | physical_nopu: 4.4256 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4820 | Classification_binary: 2.9746 | Regression_cos_phi: 1.8859 | Regression_energy: 1.5611 | Regression_eta: 1.6618 | Regression_pt: 1.7232 | Regression_sin_phi: 2.0275 | LR: 1.10e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.4583s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0035 | max_forward: 0.1952 | max_loss: 0.0038 | max_backward: 0.3195 | max_optimizer_step: 0.3092 | max_tensorboard_logging: 1.7411 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6798.7 | uss_MB: 6058.3 | child_rss_MB: 8722.6 | allocated_MB: 623.5 | reserved_MB: 27422.0 | inactive_split_MB: 864.2 +rank2 [2026-10-27 21:08:01,993] INFO : Step 72000/100000 rank2 | Train Loss: 15.2878 | Train Loss Unweighted: 4.6447 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.3643 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5735 | Regression_eta: 0.0647 | Regression_pt: 0.5353 | Regression_sin_phi: 0.0449 | Group Train Loss Unweighted (window mean): physical_pu: 3.6275 | physical_nopu: 4.4239 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5170 | Classification_binary: 3.1911 | Regression_cos_phi: 1.6451 | Regression_energy: 4.2081 | Regression_eta: 1.3010 | Regression_pt: 5.2824 | Regression_sin_phi: 1.9396 | LR: 1.05e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.5453s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0232 | max_forward: 0.1174 | max_loss: 0.0036 | max_backward: 0.0921 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 1.2606 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5641.1 | uss_MB: 4896.7 | child_rss_MB: 8682.2 | allocated_MB: 607.6 | reserved_MB: 28586.0 | inactive_split_MB: 303.9 +rank2 [2026-10-27 21:11:50,556] INFO : Step 73000/100000 rank2 | Train Loss: 12.7380 | Train Loss Unweighted: 4.2392 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.4842 | Regression_cos_phi: 0.0575 | Regression_energy: 0.3093 | Regression_eta: 0.0643 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.6309 | physical_nopu: 4.4270 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4842 | Classification_binary: 3.6662 | Regression_cos_phi: 1.8758 | Regression_energy: 1.8044 | Regression_eta: 0.8558 | Regression_pt: 2.0618 | Regression_sin_phi: 2.9693 | LR: 1.01e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.4474s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0985 | max_loss: 0.0632 | max_backward: 0.1654 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2139 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5637.1 | uss_MB: 4897.0 | child_rss_MB: 8681.3 | allocated_MB: 607.0 | reserved_MB: 28586.0 | inactive_split_MB: 227.5 +rank2 [2026-10-27 21:15:34,268] INFO : Step 74000/100000 rank2 | Train Loss: 12.5735 | Train Loss Unweighted: 4.2478 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4076 | Regression_cos_phi: 0.0429 | Regression_energy: 0.3533 | Regression_eta: 0.0699 | Regression_pt: 0.3116 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6307 | physical_nopu: 4.4218 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4562 | Classification_binary: 3.4764 | Regression_cos_phi: 1.4586 | Regression_energy: 2.0760 | Regression_eta: 1.4372 | Regression_pt: 2.4316 | Regression_sin_phi: 2.0088 | LR: 9.68e-05 | DataLoad Time: 0.1687s | Model Forward Time: 2.3800s | Phase Times: max_pre_step_barrier: 0.1687 | max_batch_to_device: 0.0008 | max_forward: 0.0990 | max_loss: 0.0516 | max_backward: 0.1766 | max_optimizer_step: 0.1812 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5626.2 | uss_MB: 4897.3 | child_rss_MB: 8634.5 | allocated_MB: 608.6 | reserved_MB: 28586.0 | inactive_split_MB: 285.4 +rank2 [2026-10-27 21:19:21,651] INFO : Step 75000/100000 rank2 | Train Loss: 15.1687 | Train Loss Unweighted: 4.6098 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.3751 | Regression_cos_phi: 0.0492 | Regression_energy: 0.5530 | Regression_eta: 0.0638 | Regression_pt: 0.5060 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4206 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4754 | Classification_binary: 3.2881 | Regression_cos_phi: 1.7962 | Regression_energy: 2.9421 | Regression_eta: 0.9770 | Regression_pt: 3.5523 | Regression_sin_phi: 2.9592 | LR: 9.27e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.4837s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1026 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.2820 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5619.2 | uss_MB: 4896.7 | child_rss_MB: 8602.7 | allocated_MB: 610.1 | reserved_MB: 28586.0 | inactive_split_MB: 259.7 +rank2 [2026-10-27 21:23:05,981] INFO : Step 76000/100000 rank2 | Train Loss: 12.2943 | Train Loss Unweighted: 4.2773 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5085 | Regression_cos_phi: 0.0476 | Regression_energy: 0.3262 | Regression_eta: 0.0732 | Regression_pt: 0.2631 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6289 | physical_nopu: 4.4209 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4314 | Classification_binary: 3.7176 | Regression_cos_phi: 1.8848 | Regression_energy: 1.6450 | Regression_eta: 0.8797 | Regression_pt: 1.8528 | Regression_sin_phi: 1.5697 | LR: 8.88e-05 | DataLoad Time: 0.0022s | Model Forward Time: 2.6226s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.1085 | max_optimizer_step: 0.1831 | max_tensorboard_logging: 1.4262 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 5617.3 | uss_MB: 4895.6 | child_rss_MB: 8601.6 | allocated_MB: 608.9 | reserved_MB: 28942.0 | inactive_split_MB: 227.1 +rank2 [2026-10-27 21:27:01,963] INFO : Step 77000/100000 rank2 | Train Loss: 13.8477 | Train Loss Unweighted: 4.4496 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4225 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4510 | Regression_eta: 0.0623 | Regression_pt: 0.4032 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6274 | physical_nopu: 4.4166 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4247 | Classification_binary: 3.3969 | Regression_cos_phi: 1.9514 | Regression_energy: 2.4195 | Regression_eta: 1.2436 | Regression_pt: 2.5634 | Regression_sin_phi: 2.0971 | LR: 8.50e-05 | DataLoad Time: 0.0018s | Model Forward Time: 6.1305s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0839 | max_forward: 0.1754 | max_loss: 0.0036 | max_backward: 0.1103 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 4.6772 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1374 | Memory: rss_MB: 5607.3 | uss_MB: 4838.5 | child_rss_MB: 8502.5 | allocated_MB: 611.0 | reserved_MB: 28942.0 | inactive_split_MB: 369.0 +rank2 [2026-10-27 21:30:52,663] INFO : Step 78000/100000 rank2 | Train Loss: 14.1981 | Train Loss Unweighted: 4.6212 | Train Loss Components: Classification: 0.0133 | Classification_binary: 3.5747 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4573 | Regression_eta: 0.0669 | Regression_pt: 0.4114 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.6241 | physical_nopu: 4.4230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4399 | Classification_binary: 3.4219 | Regression_cos_phi: 1.8210 | Regression_energy: 3.9170 | Regression_eta: 1.0966 | Regression_pt: 4.8035 | Regression_sin_phi: 1.4401 | LR: 8.13e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.0990s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1149 | max_loss: 0.0636 | max_backward: 0.1914 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5607.3 | uss_MB: 4897.1 | child_rss_MB: 8495.1 | allocated_MB: 606.6 | reserved_MB: 28942.0 | inactive_split_MB: 309.2 +rank2 [2026-10-27 21:34:49,720] INFO : Step 79000/100000 rank2 | Train Loss: 12.3648 | Train Loss Unweighted: 4.2396 | Train Loss Components: Classification: 0.0145 | Classification_binary: 3.4636 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3226 | Regression_eta: 0.0628 | Regression_pt: 0.2792 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6254 | physical_nopu: 4.4183 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4081 | Classification_binary: 3.7685 | Regression_cos_phi: 1.9290 | Regression_energy: 1.7478 | Regression_eta: 0.9381 | Regression_pt: 1.9547 | Regression_sin_phi: 1.7679 | LR: 7.78e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.3389s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0936 | max_forward: 0.1936 | max_loss: 0.0037 | max_backward: 0.1134 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 2.0079 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 5602.4 | uss_MB: 4897.2 | child_rss_MB: 8483.1 | allocated_MB: 609.2 | reserved_MB: 29222.0 | inactive_split_MB: 324.6 +rank2 [2026-10-27 21:38:37,320] INFO : Step 80000/100000 rank2 | Train Loss: 16.1833 | Train Loss Unweighted: 4.8046 | Train Loss Components: Classification: 0.0155 | Classification_binary: 3.4149 | Regression_cos_phi: 0.0453 | Regression_energy: 0.6189 | Regression_eta: 0.0801 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6233 | physical_nopu: 4.4142 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4511 | Classification_binary: 3.5843 | Regression_cos_phi: 1.6605 | Regression_energy: 3.0895 | Regression_eta: 1.2565 | Regression_pt: 3.7866 | Regression_sin_phi: 1.9458 | LR: 7.44e-05 | DataLoad Time: 0.0100s | Model Forward Time: 3.9238s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0008 | max_forward: 0.1071 | max_loss: 0.0037 | max_backward: 0.1380 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 2.6784 | max_tensorboard_flush: 0.0026 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 5596.4 | uss_MB: 4871.2 | child_rss_MB: 8436.2 | allocated_MB: 606.3 | reserved_MB: 29222.0 | inactive_split_MB: 471.7 +rank2 [2026-10-27 21:38:38,322] INFO : Running validation on rank2 +rank2 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 21:40:39,510] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 21:40:39,982] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 21:41:05,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 21:41:06,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 21:41:06,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 21:41:07,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 21:41:07,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 21:41:08,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 21:41:08,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 21:41:09,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 21:41:19,313] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 21:41:19,914] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 21:41:20,296] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 21:41:35,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 21:41:35,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 21:41:36,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 21:41:36,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 21:41:37,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 21:41:37,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 21:41:38,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 21:41:38,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 21:41:48,735] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 21:41:49,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 21:41:50,007] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 21:42:05,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 21:42:05,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 21:42:06,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 21:42:06,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 21:42:07,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 21:42:07,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 21:42:08,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 21:42:08,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 21:42:18,839] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 21:46:50,145] INFO : Step 81000/100000 rank2 | Train Loss: 11.7500 | Train Loss Unweighted: 3.6065 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.8934 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2712 | Regression_eta: 0.1074 | Regression_pt: 0.2244 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4130 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5386 | Classification_binary: 2.8238 | Regression_cos_phi: 1.7123 | Regression_energy: 1.4825 | Regression_eta: 1.7348 | Regression_pt: 1.6677 | Regression_sin_phi: 1.8830 | LR: 7.11e-05 | DataLoad Time: 0.0021s | Model Forward Time: 8.6906s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0975 | max_forward: 0.3237 | max_loss: 0.0036 | max_backward: 0.5496 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 6.3197 | max_tensorboard_flush: 0.0011 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6963.5 | uss_MB: 4835.8 | child_rss_MB: 8415.8 | allocated_MB: 619.9 | reserved_MB: 28598.0 | inactive_split_MB: 1192.4 +rank2 [2026-10-27 21:50:33,876] INFO : Step 82000/100000 rank2 | Train Loss: 13.6704 | Train Loss Unweighted: 4.5604 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0419 | Regression_energy: 0.4618 | Regression_eta: 0.0599 | Regression_pt: 0.4291 | Regression_sin_phi: 0.0407 | Group Train Loss Unweighted (window mean): physical_pu: 3.6250 | physical_nopu: 4.4152 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5605 | Classification_binary: 3.2516 | Regression_cos_phi: 1.6291 | Regression_energy: 4.0263 | Regression_eta: 1.0293 | Regression_pt: 5.1460 | Regression_sin_phi: 1.4908 | LR: 6.80e-05 | DataLoad Time: 0.0015s | Model Forward Time: 3.1355s | Phase Times: max_pre_step_barrier: 0.1835 | max_batch_to_device: 0.0874 | max_forward: 0.1976 | max_loss: 0.0036 | max_backward: 0.1170 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 1.6260 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 5659.6 | uss_MB: 4976.9 | child_rss_MB: 8356.3 | allocated_MB: 607.5 | reserved_MB: 28598.0 | inactive_split_MB: 253.8 +rank2 [2026-10-27 21:54:19,227] INFO : Step 83000/100000 rank2 | Train Loss: 12.0270 | Train Loss Unweighted: 4.2414 | Train Loss Components: Classification: 0.0092 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0499 | Regression_energy: 0.3056 | Regression_eta: 0.0671 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4141 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2819 | Classification_binary: 3.2140 | Regression_cos_phi: 1.4769 | Regression_energy: 2.3621 | Regression_eta: 0.7420 | Regression_pt: 2.9683 | Regression_sin_phi: 1.8623 | LR: 6.51e-05 | DataLoad Time: 0.0011s | Model Forward Time: 3.7074s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1966 | max_loss: 0.0138 | max_backward: 0.1124 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 2.4026 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 5659.6 | uss_MB: 4977.1 | child_rss_MB: 8350.2 | allocated_MB: 609.6 | reserved_MB: 28598.0 | inactive_split_MB: 289.5 +rank2 [2026-10-27 21:58:08,515] INFO : Step 84000/100000 rank2 | Train Loss: 12.5172 | Train Loss Unweighted: 4.2412 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.4173 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3464 | Regression_eta: 0.0637 | Regression_pt: 0.3052 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4149 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4933 | Classification_binary: 3.4193 | Regression_cos_phi: 1.4329 | Regression_energy: 1.6786 | Regression_eta: 0.8223 | Regression_pt: 2.0641 | Regression_sin_phi: 1.6160 | LR: 6.23e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.2647s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1103 | max_loss: 0.0037 | max_backward: 0.1159 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 2.0319 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5655.6 | uss_MB: 4977.8 | child_rss_MB: 8312.0 | allocated_MB: 605.9 | reserved_MB: 28598.0 | inactive_split_MB: 208.8 +rank2 [2026-10-27 22:01:46,561] INFO : Step 85000/100000 rank2 | Train Loss: 15.2814 | Train Loss Unweighted: 4.6863 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.3811 | Regression_cos_phi: 0.0432 | Regression_energy: 0.5842 | Regression_eta: 0.0608 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6173 | physical_nopu: 4.4078 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5633 | Classification_binary: 3.2991 | Regression_cos_phi: 1.5029 | Regression_energy: 3.5741 | Regression_eta: 1.0332 | Regression_pt: 4.4888 | Regression_sin_phi: 1.3233 | LR: 5.96e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.0840s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.0993 | max_loss: 0.0558 | max_backward: 0.1762 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 1.8022 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 5645.3 | uss_MB: 4977.4 | child_rss_MB: 8242.1 | allocated_MB: 606.6 | reserved_MB: 28598.0 | inactive_split_MB: 257.4 +rank2 [2026-10-27 22:05:28,665] INFO : Step 86000/100000 rank2 | Train Loss: 12.8074 | Train Loss Unweighted: 4.3373 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.5785 | Regression_cos_phi: 0.0543 | Regression_energy: 0.3116 | Regression_eta: 0.0747 | Regression_pt: 0.2515 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.6255 | physical_nopu: 4.4086 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3666 | Classification_binary: 3.4419 | Regression_cos_phi: 1.8473 | Regression_energy: 1.5406 | Regression_eta: 0.9363 | Regression_pt: 1.7132 | Regression_sin_phi: 2.0924 | LR: 5.71e-05 | DataLoad Time: 0.0025s | Model Forward Time: 3.0432s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1215 | max_loss: 0.0037 | max_backward: 0.1229 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 1.6761 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5636.3 | uss_MB: 4977.9 | child_rss_MB: 8180.8 | allocated_MB: 608.3 | reserved_MB: 29688.0 | inactive_split_MB: 271.8 +rank2 [2026-10-27 22:09:22,620] INFO : Step 87000/100000 rank2 | Train Loss: 12.9093 | Train Loss Unweighted: 4.3003 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.3807 | Regression_cos_phi: 0.0447 | Regression_energy: 0.4014 | Regression_eta: 0.0615 | Regression_pt: 0.3523 | Regression_sin_phi: 0.0424 | Group Train Loss Unweighted (window mean): physical_pu: 3.6211 | physical_nopu: 4.4120 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4985 | Classification_binary: 3.2353 | Regression_cos_phi: 1.7899 | Regression_energy: 2.1919 | Regression_eta: 0.8982 | Regression_pt: 2.4759 | Regression_sin_phi: 1.4750 | LR: 5.48e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.0855s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0203 | max_forward: 0.1299 | max_loss: 0.0038 | max_backward: 0.1175 | max_optimizer_step: 0.2192 | max_tensorboard_logging: 1.8010 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5636.1 | uss_MB: 4575.9 | child_rss_MB: 8182.7 | allocated_MB: 607.0 | reserved_MB: 29688.0 | inactive_split_MB: 231.0 +rank2 [2026-10-27 22:13:02,799] INFO : Step 88000/100000 rank2 | Train Loss: 15.2369 | Train Loss Unweighted: 4.6456 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4525 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5240 | Regression_eta: 0.0665 | Regression_pt: 0.4840 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6192 | physical_nopu: 4.4119 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4014 | Classification_binary: 3.3468 | Regression_cos_phi: 2.2941 | Regression_energy: 2.1790 | Regression_eta: 0.8605 | Regression_pt: 2.6863 | Regression_sin_phi: 1.8366 | LR: 5.26e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.1171s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0007 | max_forward: 0.1127 | max_loss: 0.0236 | max_backward: 0.1542 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 1.7956 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5636.1 | uss_MB: 4977.3 | child_rss_MB: 8181.7 | allocated_MB: 609.0 | reserved_MB: 29688.0 | inactive_split_MB: 221.0 +rank2 [2026-10-27 22:16:52,919] INFO : Step 89000/100000 rank2 | Train Loss: 12.8609 | Train Loss Unweighted: 4.2765 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.4665 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3343 | Regression_eta: 0.0681 | Regression_pt: 0.2896 | Regression_sin_phi: 0.0555 | Group Train Loss Unweighted (window mean): physical_pu: 3.6177 | physical_nopu: 4.4110 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4510 | Classification_binary: 3.5574 | Regression_cos_phi: 2.3965 | Regression_energy: 2.0665 | Regression_eta: 1.1674 | Regression_pt: 2.4971 | Regression_sin_phi: 2.3989 | LR: 5.06e-05 | DataLoad Time: 0.0026s | Model Forward Time: 3.1952s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1194 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 1.9452 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 5632.5 | uss_MB: 4977.7 | child_rss_MB: 8137.7 | allocated_MB: 612.6 | reserved_MB: 30628.0 | inactive_split_MB: 309.7 +rank2 [2026-10-27 22:20:43,711] INFO : Step 90000/100000 rank2 | Train Loss: 11.6102 | Train Loss Unweighted: 3.6127 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9177 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2661 | Regression_eta: 0.1040 | Regression_pt: 0.2155 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4109 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4774 | Classification_binary: 2.9281 | Regression_cos_phi: 1.7864 | Regression_energy: 1.4753 | Regression_eta: 1.6277 | Regression_pt: 1.6005 | Regression_sin_phi: 2.0069 | LR: 4.88e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.5771s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0110 | max_forward: 0.1856 | max_loss: 0.0145 | max_backward: 0.4526 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 1.8620 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 5624.5 | uss_MB: 4985.7 | child_rss_MB: 8062.4 | allocated_MB: 617.2 | reserved_MB: 30628.0 | inactive_split_MB: 962.2 +rank2 [2026-10-27 22:20:44,689] INFO : Running validation on rank2 +rank2 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 22:22:45,205] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 22:22:45,606] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 22:23:10,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 22:23:11,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 22:23:11,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 22:23:12,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 22:23:12,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 22:23:13,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 22:23:13,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 22:23:14,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 22:23:24,091] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 22:23:24,365] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 22:23:24,733] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 22:23:39,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 22:23:40,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 22:23:40,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 22:23:41,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 22:23:42,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 22:23:42,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 22:23:43,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 22:23:43,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 22:23:53,607] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 22:23:53,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 22:23:54,273] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 22:24:09,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 22:24:09,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 22:24:10,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 22:24:10,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 22:24:11,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 22:24:11,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 22:24:12,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 22:24:12,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 22:24:22,784] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 22:28:41,211] INFO : Step 91000/100000 rank2 | Train Loss: 12.9063 | Train Loss Unweighted: 4.3371 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5236 | Regression_cos_phi: 0.0531 | Regression_energy: 0.3431 | Regression_eta: 0.0707 | Regression_pt: 0.2829 | Regression_sin_phi: 0.0512 | Group Train Loss Unweighted (window mean): physical_pu: 3.6180 | physical_nopu: 4.4087 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4439 | Classification_binary: 3.6276 | Regression_cos_phi: 1.6361 | Regression_energy: 1.5716 | Regression_eta: 0.7883 | Regression_pt: 1.8121 | Regression_sin_phi: 1.7758 | LR: 4.71e-05 | DataLoad Time: 0.0020s | Model Forward Time: 4.8999s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0055 | max_forward: 0.1521 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 3.4850 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7066.6 | uss_MB: 6432.9 | child_rss_MB: 8020.4 | allocated_MB: 607.8 | reserved_MB: 28796.0 | inactive_split_MB: 320.2 +rank2 [2026-10-27 22:32:23,431] INFO : Step 92000/100000 rank2 | Train Loss: 15.4341 | Train Loss Unweighted: 4.7247 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4285 | Regression_cos_phi: 0.0457 | Regression_energy: 0.5811 | Regression_eta: 0.0735 | Regression_pt: 0.5375 | Regression_sin_phi: 0.0425 | Group Train Loss Unweighted (window mean): physical_pu: 3.6150 | physical_nopu: 4.4032 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4728 | Classification_binary: 3.4197 | Regression_cos_phi: 1.6726 | Regression_energy: 3.9704 | Regression_eta: 0.9273 | Regression_pt: 4.8861 | Regression_sin_phi: 1.7134 | LR: 4.57e-05 | DataLoad Time: 0.0009s | Model Forward Time: 3.3957s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.2696 | max_loss: 0.0037 | max_backward: 0.1715 | max_optimizer_step: 0.3237 | max_tensorboard_logging: 1.7768 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7051.6 | uss_MB: 6432.9 | child_rss_MB: 7931.4 | allocated_MB: 609.3 | reserved_MB: 28796.0 | inactive_split_MB: 222.8 +rank2 [2026-10-27 22:36:08,796] INFO : Step 93000/100000 rank2 | Train Loss: 13.9744 | Train Loss Unweighted: 4.4854 | Train Loss Components: Classification: 0.0118 | Classification_binary: 3.5124 | Regression_cos_phi: 0.0578 | Regression_energy: 0.4245 | Regression_eta: 0.0652 | Regression_pt: 0.3624 | Regression_sin_phi: 0.0513 | Group Train Loss Unweighted (window mean): physical_pu: 3.6205 | physical_nopu: 4.4075 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3750 | Classification_binary: 3.3727 | Regression_cos_phi: 1.9718 | Regression_energy: 1.4782 | Regression_eta: 0.9042 | Regression_pt: 1.5636 | Regression_sin_phi: 1.8138 | LR: 4.43e-05 | DataLoad Time: 0.0019s | Model Forward Time: 2.9852s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1098 | max_loss: 0.0036 | max_backward: 0.1173 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 1.7709 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5706.9 | uss_MB: 5092.9 | child_rss_MB: 7894.0 | allocated_MB: 606.4 | reserved_MB: 30394.0 | inactive_split_MB: 244.2 +rank2 [2026-10-27 22:39:53,036] INFO : Step 94000/100000 rank2 | Train Loss: 12.4397 | Train Loss Unweighted: 4.2538 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4230 | Regression_cos_phi: 0.0415 | Regression_energy: 0.3513 | Regression_eta: 0.0702 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6164 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5095 | Classification_binary: 3.3838 | Regression_cos_phi: 1.5832 | Regression_energy: 1.3195 | Regression_eta: 0.9251 | Regression_pt: 1.5940 | Regression_sin_phi: 1.5364 | LR: 4.32e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.1513s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0009 | max_forward: 0.1540 | max_loss: 0.0036 | max_backward: 0.2309 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 1.7723 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 5695.9 | uss_MB: 4753.0 | child_rss_MB: 7806.9 | allocated_MB: 607.1 | reserved_MB: 30394.0 | inactive_split_MB: 232.2 +rank2 [2026-10-27 22:43:36,954] INFO : Step 95000/100000 rank2 | Train Loss: 14.9177 | Train Loss Unweighted: 4.6108 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3911 | Regression_cos_phi: 0.0436 | Regression_energy: 0.5612 | Regression_eta: 0.0598 | Regression_pt: 0.4905 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6121 | physical_nopu: 4.4076 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3998 | Classification_binary: 3.4946 | Regression_cos_phi: 1.4297 | Regression_energy: 1.2145 | Regression_eta: 0.8139 | Regression_pt: 1.3802 | Regression_sin_phi: 1.7850 | LR: 4.22e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.0353s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1111 | max_loss: 0.0199 | max_backward: 0.1395 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.7978 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5695.9 | uss_MB: 5093.1 | child_rss_MB: 7811.5 | allocated_MB: 609.0 | reserved_MB: 30394.0 | inactive_split_MB: 285.3 +rank2 [2026-10-27 22:47:14,355] INFO : Step 96000/100000 rank2 | Train Loss: 12.4738 | Train Loss Unweighted: 4.2244 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.4884 | Regression_cos_phi: 0.0540 | Regression_energy: 0.2967 | Regression_eta: 0.0804 | Regression_pt: 0.2429 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6137 | physical_nopu: 4.4081 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3089 | Classification_binary: 3.5627 | Regression_cos_phi: 2.0179 | Regression_energy: 1.4883 | Regression_eta: 0.8678 | Regression_pt: 1.6934 | Regression_sin_phi: 2.3015 | LR: 4.14e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.1605s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0007 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.2003 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 1.8675 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5695.9 | uss_MB: 5093.2 | child_rss_MB: 7804.9 | allocated_MB: 608.1 | reserved_MB: 30394.0 | inactive_split_MB: 259.4 +rank2 [2026-10-27 22:50:52,983] INFO : Step 97000/100000 rank2 | Train Loss: 12.7922 | Train Loss Unweighted: 4.2852 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.4355 | Regression_cos_phi: 0.0455 | Regression_energy: 0.3604 | Regression_eta: 0.0703 | Regression_pt: 0.3091 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6136 | physical_nopu: 4.4015 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4864 | Classification_binary: 3.3054 | Regression_cos_phi: 1.4276 | Regression_energy: 2.0318 | Regression_eta: 1.2635 | Regression_pt: 2.4080 | Regression_sin_phi: 1.8821 | LR: 4.08e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.0105s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1110 | max_loss: 0.0174 | max_backward: 0.1239 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 1.7282 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0908 | Memory: rss_MB: 5687.4 | uss_MB: 5092.4 | child_rss_MB: 7746.9 | allocated_MB: 607.2 | reserved_MB: 30394.0 | inactive_split_MB: 265.3 +rank2 [2026-10-27 22:54:49,599] INFO : Step 98000/100000 rank2 | Train Loss: 13.8508 | Train Loss Unweighted: 4.4512 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3736 | Regression_cos_phi: 0.0443 | Regression_energy: 0.4792 | Regression_eta: 0.0606 | Regression_pt: 0.4354 | Regression_sin_phi: 0.0433 | Group Train Loss Unweighted (window mean): physical_pu: 3.6169 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4385 | Classification_binary: 3.5533 | Regression_cos_phi: 1.8837 | Regression_energy: 2.0389 | Regression_eta: 1.0548 | Regression_pt: 2.3784 | Regression_sin_phi: 1.9460 | LR: 4.04e-05 | DataLoad Time: 0.0026s | Model Forward Time: 3.0651s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0119 | max_forward: 0.1317 | max_loss: 0.0036 | max_backward: 0.1246 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.7312 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0878 | Memory: rss_MB: 5687.4 | uss_MB: 5093.6 | child_rss_MB: 7731.7 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 292.7 +rank2 [2026-10-27 22:58:30,099] INFO : Step 99000/100000 rank2 | Train Loss: 12.5729 | Train Loss Unweighted: 4.3128 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5536 | Regression_cos_phi: 0.0519 | Regression_energy: 0.3097 | Regression_eta: 0.0644 | Regression_pt: 0.2679 | Regression_sin_phi: 0.0529 | Group Train Loss Unweighted (window mean): physical_pu: 3.6154 | physical_nopu: 4.4035 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5014 | Classification_binary: 3.6824 | Regression_cos_phi: 2.0115 | Regression_energy: 1.7292 | Regression_eta: 0.9648 | Regression_pt: 2.0927 | Regression_sin_phi: 2.0821 | LR: 4.01e-05 | DataLoad Time: 0.0112s | Model Forward Time: 3.0445s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0008 | max_forward: 0.1158 | max_loss: 0.0113 | max_backward: 0.1168 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 1.7726 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 5687.4 | uss_MB: 5093.5 | child_rss_MB: 7727.1 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 283.7 +rank2 [2026-10-27 23:02:17,099] INFO : Step 100000/100000 rank2 | Train Loss: 11.9488 | Train Loss Unweighted: 3.6450 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9057 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2836 | Regression_eta: 0.1038 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6201 | physical_nopu: 4.4070 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5063 | Classification_binary: 2.9211 | Regression_cos_phi: 1.7203 | Regression_energy: 1.6384 | Regression_eta: 1.6002 | Regression_pt: 1.8971 | Regression_sin_phi: 1.9989 | LR: 4.00e-05 | DataLoad Time: 0.0035s | Model Forward Time: 6.3800s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0031 | max_forward: 0.2860 | max_loss: 0.0036 | max_backward: 0.4897 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 4.4601 | max_tensorboard_flush: 0.0009 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5687.2 | uss_MB: 4556.1 | child_rss_MB: 7711.3 | allocated_MB: 618.6 | reserved_MB: 30394.0 | inactive_split_MB: 1112.1 +rank2 [2026-10-27 23:02:18,075] INFO : Running validation on rank2 +rank2 [2026-10-27 23:02:18,075] INFO : ResumableSampler physical_pu:valid resetting +rank2 [2026-10-27 23:02:18,075] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank2 [2026-10-27 23:02:18,075] INFO : ResumableSampler physical_nopu:valid resetting +rank2 [2026-10-27 23:02:18,075] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank2 [2026-10-27 23:04:23,624] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 23:04:24,087] INFO : Running predictions on cms_pf_ttbar +rank2 [2026-10-27 23:04:49,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_2_0.parquet +rank2 [2026-10-27 23:04:50,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_2_1.parquet +rank2 [2026-10-27 23:04:50,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_2_2.parquet +rank2 [2026-10-27 23:04:51,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_2_3.parquet +rank2 [2026-10-27 23:04:51,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_2_4.parquet +rank2 [2026-10-27 23:04:52,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_2_5.parquet +rank2 [2026-10-27 23:04:52,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_2_6.parquet +rank2 [2026-10-27 23:04:53,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_2_7.parquet +rank2 [2026-10-27 23:05:03,480] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 23:05:03,812] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 23:05:04,178] INFO : Running predictions on cms_pf_qcd +rank2 [2026-10-27 23:05:19,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_2_0.parquet +rank2 [2026-10-27 23:05:20,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_2_1.parquet +rank2 [2026-10-27 23:05:20,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_2_2.parquet +rank2 [2026-10-27 23:05:21,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_2_3.parquet +rank2 [2026-10-27 23:05:21,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_2_4.parquet +rank2 [2026-10-27 23:05:22,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_2_5.parquet +rank2 [2026-10-27 23:05:22,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_2_6.parquet +rank2 [2026-10-27 23:05:23,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_2_7.parquet +rank2 [2026-10-27 23:05:33,131] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 23:05:33,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-27 23:05:34,200] INFO : Running predictions on cms_pf_ztt +rank2 [2026-10-27 23:05:49,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_2_0.parquet +rank2 [2026-10-27 23:05:50,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_2_1.parquet +rank2 [2026-10-27 23:05:50,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_2_2.parquet +rank2 [2026-10-27 23:05:51,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_2_3.parquet +rank2 [2026-10-27 23:05:51,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_2_4.parquet +rank2 [2026-10-27 23:05:52,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_2_5.parquet +rank2 [2026-10-27 23:05:52,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_2_6.parquet +rank2 [2026-10-27 23:05:53,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_2_7.parquet +rank2 [2026-10-27 23:06:03,102] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-27 23:06:03,970] INFO : Training completed. Total time on device 2: 414.793min diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.3 b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.3 new file mode 100644 index 0000000000000000000000000000000000000000..1ccb0451cf8631fd0370b190dbfe1ecf46912a77 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.3 @@ -0,0 +1,897 @@ +rank3 [2026-10-27 16:10:10,948] INFO : Initializing model with seed=12345; process seed=12348 +rank3 [2026-10-27 16:10:10,948] INFO : configured dtype=torch.bfloat16 for autocast +rank3 [2026-10-27 16:10:35,267] INFO : Instantiating model +rank3 [2026-10-27 16:10:35,267] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank3 [2026-10-27 16:10:35,267] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank3 [2026-10-27 16:10:35,296] INFO : Input encoding initialization took 0.03s +rank3 [2026-10-27 16:10:35,296] INFO : nn0 parameters: 2130944 +rank3 [2026-10-27 16:10:35,296] INFO : Initializing shared backbone layers, num_convs=3 +rank3 [2026-10-27 16:10:35,296] INFO : Initializing backbone layer 0 +rank3 [2026-10-27 16:10:35,350] INFO : Initializing backbone layer 1 +rank3 [2026-10-27 16:10:35,379] INFO : Initializing backbone layer 2 +rank3 [2026-10-27 16:10:35,408] INFO : Backbone initialization took 0.11s +rank3 [2026-10-27 16:10:35,408] INFO : backbone parameters: 15765696 +rank3 [2026-10-27 16:10:35,408] INFO : Initializing output DNNs +rank3 [2026-10-27 16:10:35,442] INFO : Output DNNs initialization took 0.03s +rank3 [2026-10-27 16:10:35,442] INFO : backbone_mode=BackboneMode.SHARED +rank3 [2026-10-27 16:10:35,442] INFO : jagged_attention enabled=False +rank3 [2026-10-27 16:10:35,442] INFO : flash_attn_varlen enabled=False +rank3 [2026-10-27 16:10:35,442] INFO : task_queries enabled=False +rank3 [2026-10-27 16:10:35,442] INFO : classification_norm parameters: 0 +rank3 [2026-10-27 16:10:35,442] INFO : regression_norm parameters: 0 +rank3 [2026-10-27 16:10:35,442] INFO : classification_query parameters: 0 +rank3 [2026-10-27 16:10:35,442] INFO : regression_query parameters: 0 +rank3 [2026-10-27 16:10:35,442] INFO : classification_readout parameters: 0 +rank3 [2026-10-27 16:10:35,442] INFO : regression_readout parameters: 0 +rank3 [2026-10-27 16:10:35,442] INFO : output_mode=elementwise +rank3 [2026-10-27 16:10:35,442] INFO : set_decoder parameters: 0 +rank3 [2026-10-27 16:10:35,442] INFO : nn_binary_particle parameters: 264706 +rank3 [2026-10-27 16:10:35,442] INFO : nn_pid parameters: 268297 +rank3 [2026-10-27 16:10:35,442] INFO : nn_pt parameters: 2113544 +rank3 [2026-10-27 16:10:35,442] INFO : nn_eta parameters: 264706 +rank3 [2026-10-27 16:10:35,443] INFO : nn_sin_phi parameters: 264706 +rank3 [2026-10-27 16:10:35,443] INFO : nn_cos_phi parameters: 264706 +rank3 [2026-10-27 16:10:35,443] INFO : nn_energy parameters: 2113544 +rank3 [2026-10-27 16:10:35,443] INFO : Total MLPF parameters: 23450849 +rank3 [2026-10-27 16:10:35,443] INFO : MLPF __init__ done +rank3 [2026-10-27 16:10:35,443] INFO : Instantiated model +rank3 [2026-10-27 16:10:35,444] INFO : Moving model to device rank=3 +rank3 [2026-10-27 16:10:35,472] INFO : Moved model to device rank=3 +rank3 [2026-10-27 16:10:35,473] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank3 [2026-10-27 16:10:35,474] INFO : Configured model for SyncBatchNorm rank=3 +rank3 [2026-10-27 16:10:36,003] INFO : Configured model for DistributedDataParallel rank=3 +rank3 [2026-10-27 16:10:36,123] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +202 module.nn_energy.nn.7.0.bias ... 0 +203 module.nn_energy.nn.7.2.weight ... 0 +204 module.nn_energy.nn.7.2.bias ... 0 +205 module.nn_energy.nn.7.4.weight ... 0 +206 module.nn_energy.nn.7.4.bias ... 0 + +[207 rows x 3 columns] +rank3 [2026-10-27 16:10:36,123] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x HEPTv2Layer( + embedding_dim=512, num_heads=16, width=512, dropout=0.0, block_size=100, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=512, out_features=512, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=512, out_features=512, bias=False) + (w_k): Linear(in_features=512, out_features=512, bias=False) + (w_v): Linear(in_features=512, out_features=512, bias=False) + (w_rpe): Linear(in_features=10, out_features=512, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=2048, bias=True) + (1): LayerNorm((2048,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=2048, out_features=512, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=512, out_features=2048, bias=False) + (up_proj): Linear(in_features=512, out_features=2048, bias=False) + (down_proj): Linear(in_features=2048, out_features=512, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank3 [2026-10-27 16:10:36,123] INFO : Trainable parameters: 23450849 +rank3 [2026-10-27 16:10:36,124] INFO : Non-trainable parameters: 22386 +rank3 [2026-10-27 16:10:36,124] INFO : Total parameters: 23473235 +rank3 [2026-10-27 16:10:36,126] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 262144 0 + module.backbone.0.attn.out_linear.bias 512 0 + module.backbone.0.attn.e2lsh.alpha 0 1632 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 262144 0 + module.backbone.0.w_k.weight 262144 0 + module.backbone.0.w_v.weight 262144 0 + module.backbone.0.w_rpe.weight 0 5120 + module.backbone.0.w_rpe.bias 0 512 +module.backbone.0.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm2.weight 512 0 + module.backbone.0.q_norm.weight 32 0 + module.backbone.0.k_norm.weight 32 0 + module.backbone.0.ff.gate_proj.weight 1048576 0 + module.backbone.0.ff.up_proj.weight 1048576 0 + module.backbone.0.ff.down_proj.weight 1048576 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 262144 0 + module.backbone.1.attn.out_linear.bias 512 0 + module.backbone.1.attn.e2lsh.alpha 0 1632 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 262144 0 + module.backbone.1.w_k.weight 262144 0 + module.backbone.1.w_v.weight 262144 0 + module.backbone.1.w_rpe.weight 0 5120 + module.backbone.1.w_rpe.bias 0 512 +module.backbone.1.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm2.weight 512 0 + module.backbone.1.q_norm.weight 32 0 + module.backbone.1.k_norm.weight 32 0 + module.backbone.1.ff.gate_proj.weight 1048576 0 + module.backbone.1.ff.up_proj.weight 1048576 0 + module.backbone.1.ff.down_proj.weight 1048576 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 262144 0 + module.backbone.2.attn.out_linear.bias 512 0 + module.backbone.2.attn.e2lsh.alpha 0 1632 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 262144 0 + module.backbone.2.w_k.weight 262144 0 + module.backbone.2.w_v.weight 262144 0 + module.backbone.2.w_rpe.weight 0 5120 + module.backbone.2.w_rpe.bias 0 512 +module.backbone.2.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm2.weight 512 0 + module.backbone.2.q_norm.weight 32 0 + module.backbone.2.k_norm.weight 32 0 + module.backbone.2.ff.gate_proj.weight 1048576 0 + module.backbone.2.ff.up_proj.weight 1048576 0 + module.backbone.2.ff.down_proj.weight 1048576 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank3 [2026-10-27 16:10:36,126] INFO : Getting dataloaders +rank3 [2026-10-27 16:10:36,126] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:43,313] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:44,039] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:44,841] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank3 [2026-10-27 16:10:44,842] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:45,823] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:46,987] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:47,781] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank3 [2026-10-27 16:10:47,781] INFO : Creating WeightedMixIterator with 2 data loaders. +rank3 [2026-10-27 16:10:47,782] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:47,952] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:48,109] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:48,265] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank3 [2026-10-27 16:10:48,265] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:48,697] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:48,855] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:10:49,000] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank3 [2026-10-27 16:10:49,000] INFO : Creating InterleavedIterator with 2 data loaders. +rank3 [2026-10-27 16:11:16,614] INFO : Got dataloaders +rank3 [2026-10-27 16:11:16,615] INFO : loader split=train rank=3 len=232058 +rank3 [2026-10-27 16:11:16,615] INFO : loader split=valid rank=3 len=48 +rank3 [2026-10-27 16:11:16,615] INFO : Starting training from step 1 to 100000 on rank 3 of 8 +rank3 [2026-10-27 16:11:16,615] INFO : Creating train_iterator +rank3 [2026-10-27 16:11:16,615] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank3 [2026-10-27 16:11:30,918] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank3 [2026-10-27 16:11:46,189] INFO : Created train_iterator +rank3 [2026-10-27 16:12:32,260] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.129867712855339, 'Classification': 0.16580097352620216, 'Regression_pt': 0.7168090478330851, 'Regression_eta': 0.335200086296536, 'Regression_sin_phi': 0.1274821354681626, 'Regression_cos_phi': 0.1371471309568733, 'Regression_energy': 0.9554258581250906}, weights={'Classification_binary': 1.0, 'Classification': 30.939912796020508, 'Regression_pt': 7.156533241271973, 'Regression_eta': 15.3038969039917, 'Regression_sin_phi': 40.23989486694336, 'Regression_cos_phi': 37.40412139892578, 'Regression_energy': 5.369194984436035} +rank3 [2026-10-27 16:15:18,322] INFO : Step 1000/100000 rank3 | Train Loss: 16.8632 | Train Loss Unweighted: 5.4934 | Train Loss Components: Classification: 0.0189 | Classification_binary: 4.1833 | Regression_cos_phi: 0.0509 | Regression_energy: 0.5806 | Regression_eta: 0.0776 | Regression_pt: 0.5302 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 5.1301 | physical_nopu: 6.0772 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5926 | Classification_binary: 4.1857 | Regression_cos_phi: 1.9111 | Regression_energy: 4.5761 | Regression_eta: 1.1802 | Regression_pt: 5.6531 | Regression_sin_phi: 2.0369 | LR: 4.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.3140s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0014 | max_forward: 0.0387 | max_loss: 0.0036 | max_backward: 0.0664 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 1.2817 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 4374.6 | uss_MB: 3102.3 | child_rss_MB: 11636.1 | allocated_MB: 609.1 | reserved_MB: 27788.0 | inactive_split_MB: 244.1 +rank3 [2026-10-27 16:18:42,007] INFO : Step 2000/100000 rank3 | Train Loss: 17.7204 | Train Loss Unweighted: 5.4242 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.9240 | Regression_cos_phi: 0.0505 | Regression_energy: 0.6758 | Regression_eta: 0.0665 | Regression_pt: 0.6366 | Regression_sin_phi: 0.0551 | Group Train Loss Unweighted (window mean): physical_pu: 4.2802 | physical_nopu: 5.3470 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5689 | Classification_binary: 3.8302 | Regression_cos_phi: 1.7824 | Regression_energy: 3.6175 | Regression_eta: 0.8647 | Regression_pt: 4.4701 | Regression_sin_phi: 2.8612 | LR: 4.00e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8063s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1422 | max_loss: 0.0036 | max_backward: 0.1717 | max_optimizer_step: 0.1501 | max_tensorboard_logging: 0.7287 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 4367.5 | uss_MB: 3102.8 | child_rss_MB: 11637.6 | allocated_MB: 607.0 | reserved_MB: 27788.0 | inactive_split_MB: 326.9 +rank3 [2026-10-27 16:22:10,507] INFO : Step 3000/100000 rank3 | Train Loss: 14.3102 | Train Loss Unweighted: 4.8875 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.8970 | Regression_cos_phi: 0.0491 | Regression_energy: 0.4277 | Regression_eta: 0.0716 | Regression_pt: 0.3769 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 4.0705 | physical_nopu: 5.1319 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4606 | Classification_binary: 3.9146 | Regression_cos_phi: 1.6185 | Regression_energy: 2.4643 | Regression_eta: 1.1812 | Regression_pt: 2.6857 | Regression_sin_phi: 1.6876 | LR: 3.99e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.0072s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0782 | max_loss: 0.0037 | max_backward: 0.0505 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 1.0271 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4295.2 | uss_MB: 3034.9 | child_rss_MB: 11625.5 | allocated_MB: 607.6 | reserved_MB: 28576.0 | inactive_split_MB: 261.0 +rank3 [2026-10-27 16:25:44,194] INFO : Step 4000/100000 rank3 | Train Loss: 12.6160 | Train Loss Unweighted: 3.9080 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.1145 | Regression_cos_phi: 0.0484 | Regression_energy: 0.3088 | Regression_eta: 0.1117 | Regression_pt: 0.2597 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.9908 | physical_nopu: 5.0219 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4927 | Classification_binary: 3.1192 | Regression_cos_phi: 1.9165 | Regression_energy: 1.6272 | Regression_eta: 1.6643 | Regression_pt: 1.8056 | Regression_sin_phi: 1.9568 | LR: 3.99e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4274s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0032 | max_forward: 0.1365 | max_loss: 0.0036 | max_backward: 0.2496 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 1.0566 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 4292.4 | uss_MB: 3043.1 | child_rss_MB: 11491.5 | allocated_MB: 617.3 | reserved_MB: 28576.0 | inactive_split_MB: 1049.1 +rank3 [2026-10-27 16:29:29,026] INFO : Step 5000/100000 rank3 | Train Loss: 13.7309 | Train Loss Unweighted: 4.7575 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.8762 | Regression_cos_phi: 0.0508 | Regression_energy: 0.3661 | Regression_eta: 0.0922 | Regression_pt: 0.3105 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.9453 | physical_nopu: 4.9561 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4567 | Classification_binary: 3.8609 | Regression_cos_phi: 1.8427 | Regression_energy: 2.0130 | Regression_eta: 1.3511 | Regression_pt: 2.2539 | Regression_sin_phi: 1.7848 | LR: 3.98e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.0873s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.0008 | max_forward: 0.0371 | max_loss: 0.0036 | max_backward: 0.0568 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 2.1174 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4241.6 | uss_MB: 3036.2 | child_rss_MB: 11146.9 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 248.7 +rank3 [2026-10-27 16:33:03,739] INFO : Step 6000/100000 rank3 | Train Loss: 15.2316 | Train Loss Unweighted: 4.9080 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.7479 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5147 | Regression_eta: 0.0679 | Regression_pt: 0.4609 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.9059 | physical_nopu: 4.9043 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4925 | Classification_binary: 3.8258 | Regression_cos_phi: 2.2159 | Regression_energy: 2.5059 | Regression_eta: 1.0591 | Regression_pt: 2.9518 | Regression_sin_phi: 1.9135 | LR: 3.97e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.0045s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0421 | max_loss: 0.0036 | max_backward: 0.0620 | max_optimizer_step: 0.1489 | max_tensorboard_logging: 2.0592 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4216.9 | uss_MB: 3036.3 | child_rss_MB: 11044.6 | allocated_MB: 610.3 | reserved_MB: 28576.0 | inactive_split_MB: 254.0 +rank3 [2026-10-27 16:36:49,903] INFO : Step 7000/100000 rank3 | Train Loss: 14.7215 | Train Loss Unweighted: 4.8917 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8534 | Regression_cos_phi: 0.0503 | Regression_energy: 0.4508 | Regression_eta: 0.0791 | Regression_pt: 0.3921 | Regression_sin_phi: 0.0547 | Group Train Loss Unweighted (window mean): physical_pu: 3.8763 | physical_nopu: 4.8627 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3195 | Classification_binary: 3.8555 | Regression_cos_phi: 1.9676 | Regression_energy: 1.6349 | Regression_eta: 0.9155 | Regression_pt: 1.8389 | Regression_sin_phi: 2.0191 | LR: 3.96e-04 | DataLoad Time: 0.0012s | Model Forward Time: 4.5452s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0348 | max_loss: 0.0036 | max_backward: 0.0535 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 3.5409 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4194.7 | uss_MB: 3033.5 | child_rss_MB: 10983.9 | allocated_MB: 609.0 | reserved_MB: 28576.0 | inactive_split_MB: 216.2 +rank3 [2026-10-27 16:40:34,139] INFO : Step 8000/100000 rank3 | Train Loss: 14.2560 | Train Loss Unweighted: 4.8042 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.8045 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4333 | Regression_eta: 0.0601 | Regression_pt: 0.3883 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.8518 | physical_nopu: 4.8294 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5071 | Classification_binary: 3.8759 | Regression_cos_phi: 1.7791 | Regression_energy: 2.3514 | Regression_eta: 0.9091 | Regression_pt: 2.7343 | Regression_sin_phi: 1.9317 | LR: 3.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 4.8860s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1342 | max_loss: 0.0036 | max_backward: 0.0500 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 3.8645 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4176.9 | uss_MB: 3036.4 | child_rss_MB: 10934.3 | allocated_MB: 606.6 | reserved_MB: 28576.0 | inactive_split_MB: 255.4 +rank3 [2026-10-27 16:44:13,307] INFO : Step 9000/100000 rank3 | Train Loss: 15.9274 | Train Loss Unweighted: 4.9361 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.6437 | Regression_cos_phi: 0.0472 | Regression_energy: 0.5701 | Regression_eta: 0.0733 | Regression_pt: 0.5360 | Regression_sin_phi: 0.0496 | Group Train Loss Unweighted (window mean): physical_pu: 3.8376 | physical_nopu: 4.8057 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5175 | Classification_binary: 3.4967 | Regression_cos_phi: 1.7392 | Regression_energy: 3.3338 | Regression_eta: 1.1403 | Regression_pt: 3.9358 | Regression_sin_phi: 2.0586 | LR: 3.93e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.2109s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1320 | max_loss: 0.0037 | max_backward: 0.0555 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 2.0797 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 4157.5 | uss_MB: 3035.9 | child_rss_MB: 10877.2 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 321.4 +rank3 [2026-10-27 16:48:05,432] INFO : Step 10000/100000 rank3 | Train Loss: 13.0514 | Train Loss Unweighted: 4.5682 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.7678 | Regression_cos_phi: 0.0520 | Regression_energy: 0.3372 | Regression_eta: 0.0736 | Regression_pt: 0.2745 | Regression_sin_phi: 0.0521 | Group Train Loss Unweighted (window mean): physical_pu: 3.8237 | physical_nopu: 4.7797 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3471 | Classification_binary: 3.8209 | Regression_cos_phi: 2.0889 | Regression_energy: 2.0994 | Regression_eta: 1.0347 | Regression_pt: 2.3773 | Regression_sin_phi: 2.4676 | LR: 3.91e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.2566s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.0379 | max_loss: 0.0040 | max_backward: 0.0556 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 2.2516 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4149.4 | uss_MB: 3036.1 | child_rss_MB: 10872.4 | allocated_MB: 607.9 | reserved_MB: 28576.0 | inactive_split_MB: 258.7 +rank3 [2026-10-27 16:48:06,334] INFO : Running validation on rank3 +rank3 [2026-10-27 16:48:06,334] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 16:48:06,334] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 16:48:06,334] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 16:48:06,334] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 16:50:06,283] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:50:06,771] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 16:50:34,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 16:50:35,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 16:50:35,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 16:50:36,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 16:50:37,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 16:50:37,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 16:50:38,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 16:50:38,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 16:50:48,683] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-27 16:50:48,968] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:50:49,337] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 16:51:04,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 16:51:05,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 16:51:05,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 16:51:06,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 16:51:06,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 16:51:07,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 16:51:07,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 16:51:08,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 16:51:18,495] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 16:51:18,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 16:51:19,153] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 16:51:34,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 16:51:34,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 16:51:35,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 16:51:35,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 16:51:36,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 16:51:37,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 16:51:37,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 16:51:38,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 16:51:48,075] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 16:55:58,693] INFO : Step 11000/100000 rank3 | Train Loss: 14.4095 | Train Loss Unweighted: 4.7539 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.7591 | Regression_cos_phi: 0.0522 | Regression_energy: 0.4228 | Regression_eta: 0.0709 | Regression_pt: 0.3796 | Regression_sin_phi: 0.0518 | Group Train Loss Unweighted (window mean): physical_pu: 3.8077 | physical_nopu: 4.7444 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6460 | Classification_binary: 3.7666 | Regression_cos_phi: 1.9900 | Regression_energy: 2.5074 | Regression_eta: 1.1190 | Regression_pt: 3.0789 | Regression_sin_phi: 2.1962 | LR: 3.89e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.1995s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1127 | max_loss: 0.0074 | max_backward: 0.1410 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 1.9404 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6580.9 | uss_MB: 5462.7 | child_rss_MB: 10900.1 | allocated_MB: 610.6 | reserved_MB: 27422.0 | inactive_split_MB: 253.4 +rank3 [2026-10-27 16:59:46,314] INFO : Step 12000/100000 rank3 | Train Loss: 16.4725 | Train Loss Unweighted: 5.0863 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.5958 | Regression_cos_phi: 0.0410 | Regression_energy: 0.6842 | Regression_eta: 0.0610 | Regression_pt: 0.6484 | Regression_sin_phi: 0.0394 | Group Train Loss Unweighted (window mean): physical_pu: 3.8013 | physical_nopu: 4.7308 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4850 | Classification_binary: 3.3198 | Regression_cos_phi: 0.9312 | Regression_energy: 5.4744 | Regression_eta: 0.7139 | Regression_pt: 7.1070 | Regression_sin_phi: 1.2531 | LR: 3.87e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.5369s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1292 | max_loss: 0.0037 | max_backward: 0.1951 | max_optimizer_step: 0.2940 | max_tensorboard_logging: 2.1160 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6552.9 | uss_MB: 5462.9 | child_rss_MB: 10800.4 | allocated_MB: 609.5 | reserved_MB: 27422.0 | inactive_split_MB: 347.2 +rank3 [2026-10-27 17:03:40,557] INFO : Step 13000/100000 rank3 | Train Loss: 12.9993 | Train Loss Unweighted: 4.5752 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.7655 | Regression_cos_phi: 0.0483 | Regression_energy: 0.3375 | Regression_eta: 0.0674 | Regression_pt: 0.2918 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7898 | physical_nopu: 4.7063 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2650 | Classification_binary: 3.6910 | Regression_cos_phi: 1.6010 | Regression_energy: 1.4973 | Regression_eta: 0.8141 | Regression_pt: 1.8210 | Regression_sin_phi: 1.6043 | LR: 3.85e-04 | DataLoad Time: 0.0088s | Model Forward Time: 3.0969s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1327 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.9573 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5436.1 | uss_MB: 4383.7 | child_rss_MB: 10499.2 | allocated_MB: 607.2 | reserved_MB: 28584.0 | inactive_split_MB: 340.8 +rank3 [2026-10-27 17:07:23,686] INFO : Step 14000/100000 rank3 | Train Loss: 12.1035 | Train Loss Unweighted: 3.7686 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.0394 | Regression_cos_phi: 0.0492 | Regression_energy: 0.2767 | Regression_eta: 0.1064 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0483 | Group Train Loss Unweighted (window mean): physical_pu: 3.7803 | physical_nopu: 4.6878 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4954 | Classification_binary: 2.9594 | Regression_cos_phi: 1.8441 | Regression_energy: 1.4455 | Regression_eta: 1.5983 | Regression_pt: 1.6066 | Regression_sin_phi: 1.9147 | LR: 3.83e-04 | DataLoad Time: 0.0019s | Model Forward Time: 5.5486s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0034 | max_forward: 0.2002 | max_loss: 0.0036 | max_backward: 0.3668 | max_optimizer_step: 0.2700 | max_tensorboard_logging: 3.9098 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0824 | Memory: rss_MB: 5443.1 | uss_MB: 4393.2 | child_rss_MB: 10497.0 | allocated_MB: 618.5 | reserved_MB: 28584.0 | inactive_split_MB: 893.5 +rank3 [2026-10-27 17:11:07,917] INFO : Step 15000/100000 rank3 | Train Loss: 13.2483 | Train Loss Unweighted: 4.5172 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.6965 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3441 | Regression_eta: 0.0751 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7702 | physical_nopu: 4.6706 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4853 | Classification_binary: 3.7157 | Regression_cos_phi: 2.0975 | Regression_energy: 1.3876 | Regression_eta: 1.1449 | Regression_pt: 1.5495 | Regression_sin_phi: 1.7002 | LR: 3.80e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.3450s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0879 | max_loss: 0.0036 | max_backward: 0.0859 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 2.2540 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0190 | Memory: rss_MB: 5437.1 | uss_MB: 4385.0 | child_rss_MB: 10498.2 | allocated_MB: 605.9 | reserved_MB: 28584.0 | inactive_split_MB: 314.1 +rank3 [2026-10-27 17:14:55,830] INFO : Step 16000/100000 rank3 | Train Loss: 14.0060 | Train Loss Unweighted: 4.5724 | Train Loss Components: Classification: 0.0150 | Classification_binary: 3.5827 | Regression_cos_phi: 0.0469 | Regression_energy: 0.4276 | Regression_eta: 0.0760 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0517 | Group Train Loss Unweighted (window mean): physical_pu: 3.7654 | physical_nopu: 4.6500 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4205 | Classification_binary: 3.5502 | Regression_cos_phi: 1.5038 | Regression_energy: 2.3322 | Regression_eta: 1.2974 | Regression_pt: 2.7667 | Regression_sin_phi: 1.7025 | LR: 3.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.4625s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0984 | max_forward: 0.1749 | max_loss: 0.0039 | max_backward: 0.0976 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 2.1733 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5435.1 | uss_MB: 4384.3 | child_rss_MB: 10495.1 | allocated_MB: 607.6 | reserved_MB: 28978.0 | inactive_split_MB: 330.4 +rank3 [2026-10-27 17:18:33,642] INFO : Step 17000/100000 rank3 | Train Loss: 14.2557 | Train Loss Unweighted: 4.6663 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6292 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4540 | Regression_eta: 0.0715 | Regression_pt: 0.3998 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.7576 | physical_nopu: 4.6430 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3648 | Classification_binary: 3.7088 | Regression_cos_phi: 1.7762 | Regression_energy: 2.5118 | Regression_eta: 1.0569 | Regression_pt: 2.7867 | Regression_sin_phi: 2.2590 | LR: 3.75e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.9706s | Phase Times: max_pre_step_barrier: 0.5705 | max_batch_to_device: 0.0043 | max_forward: 0.0887 | max_loss: 0.0037 | max_backward: 0.1154 | max_optimizer_step: 0.2542 | max_tensorboard_logging: 2.1801 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5421.0 | uss_MB: 4384.2 | child_rss_MB: 10406.9 | allocated_MB: 609.7 | reserved_MB: 29500.0 | inactive_split_MB: 302.1 +rank3 [2026-10-27 17:22:18,273] INFO : Step 18000/100000 rank3 | Train Loss: 13.2896 | Train Loss Unweighted: 4.5062 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6504 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3607 | Regression_eta: 0.0732 | Regression_pt: 0.3062 | Regression_sin_phi: 0.0530 | Group Train Loss Unweighted (window mean): physical_pu: 3.7500 | physical_nopu: 4.6286 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4523 | Classification_binary: 3.8580 | Regression_cos_phi: 2.5143 | Regression_energy: 2.5685 | Regression_eta: 1.5080 | Regression_pt: 2.6232 | Regression_sin_phi: 2.5968 | LR: 3.72e-04 | DataLoad Time: 0.0013s | Model Forward Time: 10.0057s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0832 | max_loss: 0.0037 | max_backward: 0.0999 | max_optimizer_step: 0.1725 | max_tensorboard_logging: 8.8254 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 5416.7 | uss_MB: 4384.0 | child_rss_MB: 10363.7 | allocated_MB: 608.8 | reserved_MB: 29500.0 | inactive_split_MB: 196.1 +rank3 [2026-10-27 17:25:51,407] INFO : Step 19000/100000 rank3 | Train Loss: 15.2449 | Train Loss Unweighted: 4.8480 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5669 | Regression_cos_phi: 0.0399 | Regression_energy: 0.5792 | Regression_eta: 0.0602 | Regression_pt: 0.5415 | Regression_sin_phi: 0.0443 | Group Train Loss Unweighted (window mean): physical_pu: 3.7399 | physical_nopu: 4.6217 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4291 | Classification_binary: 3.5451 | Regression_cos_phi: 1.5069 | Regression_energy: 2.1349 | Regression_eta: 0.9071 | Regression_pt: 2.4815 | Regression_sin_phi: 1.6587 | LR: 3.69e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.1644s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0008 | max_forward: 0.0818 | max_loss: 0.0762 | max_backward: 0.1759 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 1.9822 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5410.7 | uss_MB: 4384.1 | child_rss_MB: 10330.8 | allocated_MB: 607.1 | reserved_MB: 29500.0 | inactive_split_MB: 236.2 +rank3 [2026-10-27 17:29:45,026] INFO : Step 20000/100000 rank3 | Train Loss: 13.7275 | Train Loss Unweighted: 4.6495 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.7120 | Regression_cos_phi: 0.0510 | Regression_energy: 0.4061 | Regression_eta: 0.0646 | Regression_pt: 0.3531 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.7387 | physical_nopu: 4.6038 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4711 | Classification_binary: 3.7108 | Regression_cos_phi: 1.6996 | Regression_energy: 1.8363 | Regression_eta: 0.7454 | Regression_pt: 2.0810 | Regression_sin_phi: 2.2275 | LR: 3.66e-04 | DataLoad Time: 0.0027s | Model Forward Time: 5.4931s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0007 | max_forward: 0.0851 | max_loss: 0.0039 | max_backward: 0.2016 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 4.2823 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5411.1 | uss_MB: 4383.8 | child_rss_MB: 10307.1 | allocated_MB: 609.5 | reserved_MB: 29500.0 | inactive_split_MB: 252.8 +rank3 [2026-10-27 17:29:45,969] INFO : Running validation on rank3 +rank3 [2026-10-27 17:29:45,969] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 17:29:45,969] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 17:29:45,969] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 17:31:42,604] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 17:31:43,254] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 17:32:09,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 17:32:10,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 17:32:10,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 17:32:11,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 17:32:11,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 17:32:12,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 17:32:12,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 17:32:13,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 17:32:23,175] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 17:32:23,635] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 17:32:24,023] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 17:32:39,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 17:32:39,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 17:32:40,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 17:32:40,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 17:32:41,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 17:32:41,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 17:32:42,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 17:32:42,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 17:32:52,910] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 17:32:53,432] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 17:32:53,824] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 17:33:09,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 17:33:09,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 17:33:09,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 17:33:10,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 17:33:10,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 17:33:11,495] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 17:33:11,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 17:33:12,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 17:33:22,532] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 17:37:33,418] INFO : Step 21000/100000 rank3 | Train Loss: 13.1647 | Train Loss Unweighted: 4.3967 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.5106 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3742 | Regression_eta: 0.0796 | Regression_pt: 0.3229 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.7328 | physical_nopu: 4.5933 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4862 | Classification_binary: 3.6655 | Regression_cos_phi: 1.8737 | Regression_energy: 1.9226 | Regression_eta: 1.4796 | Regression_pt: 2.1589 | Regression_sin_phi: 1.9864 | LR: 3.62e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.6671s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0483 | max_forward: 0.1678 | max_loss: 0.0037 | max_backward: 0.1430 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 2.3569 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5516.5 | uss_MB: 4216.2 | child_rss_MB: 10241.6 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 238.6 +rank3 [2026-10-27 17:41:10,019] INFO : Step 22000/100000 rank3 | Train Loss: 15.3802 | Train Loss Unweighted: 4.7914 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.5311 | Regression_cos_phi: 0.0475 | Regression_energy: 0.5593 | Regression_eta: 0.0730 | Regression_pt: 0.5219 | Regression_sin_phi: 0.0434 | Group Train Loss Unweighted (window mean): physical_pu: 3.7290 | physical_nopu: 4.5858 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4592 | Classification_binary: 3.4303 | Regression_cos_phi: 1.5840 | Regression_energy: 2.7111 | Regression_eta: 0.9906 | Regression_pt: 3.4225 | Regression_sin_phi: 1.4655 | LR: 3.59e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.1741s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0905 | max_loss: 0.0038 | max_backward: 0.1361 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 2.0169 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5517.5 | uss_MB: 4498.8 | child_rss_MB: 10247.2 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 321.0 +rank3 [2026-10-27 17:44:47,650] INFO : Step 23000/100000 rank3 | Train Loss: 12.9848 | Train Loss Unweighted: 4.5226 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.7377 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3326 | Regression_eta: 0.0739 | Regression_pt: 0.2617 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.7288 | physical_nopu: 4.5774 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3602 | Classification_binary: 3.7052 | Regression_cos_phi: 1.9538 | Regression_energy: 1.7153 | Regression_eta: 1.1839 | Regression_pt: 1.8739 | Regression_sin_phi: 2.6383 | LR: 3.55e-04 | DataLoad Time: 0.0063s | Model Forward Time: 3.1593s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0994 | max_loss: 0.0039 | max_backward: 0.1244 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.9838 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 5517.5 | uss_MB: 4498.0 | child_rss_MB: 10251.2 | allocated_MB: 608.2 | reserved_MB: 29422.0 | inactive_split_MB: 218.5 +rank3 [2026-10-27 17:48:44,022] INFO : Step 24000/100000 rank3 | Train Loss: 16.3896 | Train Loss Unweighted: 4.8949 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4510 | Regression_cos_phi: 0.0465 | Regression_energy: 0.6613 | Regression_eta: 0.0617 | Regression_pt: 0.6132 | Regression_sin_phi: 0.0453 | Group Train Loss Unweighted (window mean): physical_pu: 3.7170 | physical_nopu: 4.5767 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5567 | Classification_binary: 3.3413 | Regression_cos_phi: 2.0660 | Regression_energy: 3.1804 | Regression_eta: 0.8666 | Regression_pt: 3.8757 | Regression_sin_phi: 2.2234 | LR: 3.51e-04 | DataLoad Time: 0.0028s | Model Forward Time: 3.1023s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0398 | max_forward: 0.1314 | max_loss: 0.0037 | max_backward: 0.0994 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 1.8290 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5518.7 | uss_MB: 4498.4 | child_rss_MB: 10238.8 | allocated_MB: 606.3 | reserved_MB: 29422.0 | inactive_split_MB: 288.6 +rank3 [2026-10-27 17:52:29,098] INFO : Step 25000/100000 rank3 | Train Loss: 13.0062 | Train Loss Unweighted: 4.4048 | Train Loss Components: Classification: 0.0114 | Classification_binary: 3.6384 | Regression_cos_phi: 0.0552 | Regression_energy: 0.3153 | Regression_eta: 0.0680 | Regression_pt: 0.2574 | Regression_sin_phi: 0.0590 | Group Train Loss Unweighted (window mean): physical_pu: 3.7138 | physical_nopu: 4.5644 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3730 | Classification_binary: 3.6833 | Regression_cos_phi: 2.0198 | Regression_energy: 1.8057 | Regression_eta: 1.1035 | Regression_pt: 1.9817 | Regression_sin_phi: 2.6724 | LR: 3.47e-04 | DataLoad Time: 0.0015s | Model Forward Time: 2.5993s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0007 | max_forward: 0.1394 | max_loss: 0.0036 | max_backward: 0.2004 | max_optimizer_step: 0.1603 | max_tensorboard_logging: 1.3791 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5517.7 | uss_MB: 4498.3 | child_rss_MB: 10255.6 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 213.9 +rank3 [2026-10-27 17:56:10,619] INFO : Step 26000/100000 rank3 | Train Loss: 13.5469 | Train Loss Unweighted: 4.4523 | Train Loss Components: Classification: 0.0185 | Classification_binary: 3.5246 | Regression_cos_phi: 0.0498 | Regression_energy: 0.3935 | Regression_eta: 0.0732 | Regression_pt: 0.3461 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5624 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.7478 | Classification_binary: 3.3792 | Regression_cos_phi: 2.2876 | Regression_energy: 2.1980 | Regression_eta: 0.8753 | Regression_pt: 2.7488 | Regression_sin_phi: 1.8664 | LR: 3.43e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.1209s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0838 | max_forward: 0.1679 | max_loss: 0.0039 | max_backward: 0.0991 | max_optimizer_step: 0.2674 | max_tensorboard_logging: 1.6325 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5515.7 | uss_MB: 4498.4 | child_rss_MB: 10248.9 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 201.9 +rank3 [2026-10-27 17:59:48,627] INFO : Step 27000/100000 rank3 | Train Loss: 17.1780 | Train Loss Unweighted: 5.0531 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.5146 | Regression_cos_phi: 0.0448 | Regression_energy: 0.6955 | Regression_eta: 0.0778 | Regression_pt: 0.6583 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5579 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5194 | Classification_binary: 3.3591 | Regression_cos_phi: 1.3741 | Regression_energy: 5.1703 | Regression_eta: 0.9900 | Regression_pt: 6.7512 | Regression_sin_phi: 1.6499 | LR: 3.39e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.4474s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1019 | max_loss: 0.0489 | max_backward: 0.1596 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5507.7 | uss_MB: 4499.5 | child_rss_MB: 10244.8 | allocated_MB: 611.7 | reserved_MB: 30248.0 | inactive_split_MB: 336.3 +rank3 [2026-10-27 18:03:26,753] INFO : Step 28000/100000 rank3 | Train Loss: 12.6721 | Train Loss Unweighted: 4.3436 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5550 | Regression_cos_phi: 0.0488 | Regression_energy: 0.3264 | Regression_eta: 0.0754 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7057 | physical_nopu: 4.5482 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3569 | Classification_binary: 3.7127 | Regression_cos_phi: 1.9321 | Regression_energy: 1.8981 | Regression_eta: 1.3005 | Regression_pt: 2.0406 | Regression_sin_phi: 2.3831 | LR: 3.35e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.0158s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0898 | max_loss: 0.0057 | max_backward: 0.1164 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.8021 | max_tensorboard_flush: 0.0040 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 5489.8 | uss_MB: 4496.9 | child_rss_MB: 10097.2 | allocated_MB: 606.6 | reserved_MB: 30248.0 | inactive_split_MB: 278.7 +rank3 [2026-10-27 18:07:25,425] INFO : Step 29000/100000 rank3 | Train Loss: 14.8403 | Train Loss Unweighted: 4.5753 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.4458 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4901 | Regression_eta: 0.0767 | Regression_pt: 0.4459 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.7068 | physical_nopu: 4.5439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5037 | Classification_binary: 3.2563 | Regression_cos_phi: 1.7904 | Regression_energy: 3.8667 | Regression_eta: 1.1178 | Regression_pt: 5.0446 | Regression_sin_phi: 1.9458 | LR: 3.30e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.3381s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1464 | max_loss: 0.0037 | max_backward: 0.0935 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 1.1535 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5489.8 | uss_MB: 4498.4 | child_rss_MB: 10094.9 | allocated_MB: 607.4 | reserved_MB: 30248.0 | inactive_split_MB: 313.3 +rank3 [2026-10-27 18:11:12,494] INFO : Step 30000/100000 rank3 | Train Loss: 13.2916 | Train Loss Unweighted: 4.4954 | Train Loss Components: Classification: 0.0126 | Classification_binary: 3.5989 | Regression_cos_phi: 0.0482 | Regression_energy: 0.3865 | Regression_eta: 0.0707 | Regression_pt: 0.3291 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7005 | physical_nopu: 4.5378 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3667 | Classification_binary: 3.6145 | Regression_cos_phi: 1.4621 | Regression_energy: 2.9056 | Regression_eta: 0.9903 | Regression_pt: 3.4614 | Regression_sin_phi: 2.0525 | LR: 3.26e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.7225s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0805 | max_loss: 0.0038 | max_backward: 0.0926 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 1.5279 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 5479.8 | uss_MB: 4498.3 | child_rss_MB: 10016.6 | allocated_MB: 608.2 | reserved_MB: 30760.0 | inactive_split_MB: 232.9 +rank3 [2026-10-27 18:11:13,431] INFO : Running validation on rank3 +rank3 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 18:13:07,586] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 18:13:08,047] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 18:13:34,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 18:13:34,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 18:13:35,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 18:13:35,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 18:13:36,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 18:13:36,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 18:13:37,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 18:13:37,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 18:13:47,995] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 18:13:48,509] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 18:13:49,355] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 18:14:04,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 18:14:04,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 18:14:05,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 18:14:05,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 18:14:06,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 18:14:06,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 18:14:07,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 18:14:07,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 18:14:17,902] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 18:14:18,746] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 18:14:19,219] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 18:14:34,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 18:14:34,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 18:14:35,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 18:14:35,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 18:14:36,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 18:14:36,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 18:14:37,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 18:14:37,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 18:14:47,731] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 18:19:00,303] INFO : Step 31000/100000 rank3 | Train Loss: 12.7750 | Train Loss Unweighted: 4.3889 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5406 | Regression_cos_phi: 0.0465 | Regression_energy: 0.3622 | Regression_eta: 0.0700 | Regression_pt: 0.3101 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6964 | physical_nopu: 4.5360 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4687 | Classification_binary: 3.6438 | Regression_cos_phi: 2.0941 | Regression_energy: 1.8673 | Regression_eta: 1.0848 | Regression_pt: 2.2270 | Regression_sin_phi: 2.3120 | LR: 3.21e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.3236s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.2346 | max_loss: 0.0037 | max_backward: 0.1255 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 1.8384 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1395 | Memory: rss_MB: 6773.5 | uss_MB: 5791.8 | child_rss_MB: 10011.8 | allocated_MB: 609.3 | reserved_MB: 29142.0 | inactive_split_MB: 278.5 +rank3 [2026-10-27 18:22:44,787] INFO : Step 32000/100000 rank3 | Train Loss: 14.3617 | Train Loss Unweighted: 4.6225 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4934 | Regression_cos_phi: 0.0448 | Regression_energy: 0.5043 | Regression_eta: 0.0718 | Regression_pt: 0.4507 | Regression_sin_phi: 0.0414 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5314 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5341 | Classification_binary: 3.6405 | Regression_cos_phi: 2.0617 | Regression_energy: 2.3160 | Regression_eta: 1.0465 | Regression_pt: 2.7654 | Regression_sin_phi: 2.1936 | LR: 3.16e-04 | DataLoad Time: 0.0109s | Model Forward Time: 2.5617s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1373 | max_loss: 0.0041 | max_backward: 0.2040 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 1.2065 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5556.1 | uss_MB: 4574.8 | child_rss_MB: 10008.9 | allocated_MB: 606.1 | reserved_MB: 29142.0 | inactive_split_MB: 511.2 +rank3 [2026-10-27 18:26:33,025] INFO : Step 33000/100000 rank3 | Train Loss: 12.0000 | Train Loss Unweighted: 3.7115 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9674 | Regression_cos_phi: 0.0474 | Regression_energy: 0.2863 | Regression_eta: 0.1056 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4735 | Classification_binary: 2.9199 | Regression_cos_phi: 1.7520 | Regression_energy: 1.4361 | Regression_eta: 1.6632 | Regression_pt: 1.5920 | Regression_sin_phi: 1.9098 | LR: 3.12e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.5985s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0030 | max_forward: 0.1619 | max_loss: 0.0119 | max_backward: 0.2845 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.1927 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5557.1 | uss_MB: 4556.3 | child_rss_MB: 10013.2 | allocated_MB: 623.7 | reserved_MB: 29572.0 | inactive_split_MB: 1441.5 +rank3 [2026-10-27 18:30:15,971] INFO : Step 34000/100000 rank3 | Train Loss: 15.0627 | Train Loss Unweighted: 4.6899 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4670 | Regression_cos_phi: 0.0454 | Regression_energy: 0.5438 | Regression_eta: 0.0626 | Regression_pt: 0.5082 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6889 | physical_nopu: 4.5215 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5102 | Classification_binary: 3.4437 | Regression_cos_phi: 1.8738 | Regression_energy: 3.0522 | Regression_eta: 0.8568 | Regression_pt: 3.6736 | Regression_sin_phi: 1.6035 | LR: 3.07e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.3753s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0007 | max_forward: 0.1346 | max_loss: 0.0037 | max_backward: 0.1938 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1506 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5552.6 | uss_MB: 4573.6 | child_rss_MB: 9984.9 | allocated_MB: 609.6 | reserved_MB: 29572.0 | inactive_split_MB: 230.4 +rank3 [2026-10-27 18:33:55,272] INFO : Step 35000/100000 rank3 | Train Loss: 12.2019 | Train Loss Unweighted: 4.3540 | Train Loss Components: Classification: 0.0089 | Classification_binary: 3.6426 | Regression_cos_phi: 0.0532 | Regression_energy: 0.2965 | Regression_eta: 0.0664 | Regression_pt: 0.2370 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6854 | physical_nopu: 4.5184 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2065 | Classification_binary: 3.4120 | Regression_cos_phi: 1.8957 | Regression_energy: 1.5451 | Regression_eta: 0.9427 | Regression_pt: 1.7067 | Regression_sin_phi: 1.7755 | LR: 3.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.6469s | Phase Times: max_pre_step_barrier: 0.0511 | max_batch_to_device: 0.0857 | max_forward: 0.1832 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 1.2656 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 5548.6 | uss_MB: 4574.4 | child_rss_MB: 9918.7 | allocated_MB: 608.2 | reserved_MB: 29572.0 | inactive_split_MB: 245.8 +rank3 [2026-10-27 18:37:33,850] INFO : Step 36000/100000 rank3 | Train Loss: 12.7130 | Train Loss Unweighted: 4.3877 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.5479 | Regression_cos_phi: 0.0433 | Regression_energy: 0.3521 | Regression_eta: 0.0636 | Regression_pt: 0.3167 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.6860 | physical_nopu: 4.5137 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4840 | Classification_binary: 3.6515 | Regression_cos_phi: 1.3917 | Regression_energy: 1.7250 | Regression_eta: 0.8731 | Regression_pt: 2.0983 | Regression_sin_phi: 1.6395 | LR: 2.97e-04 | DataLoad Time: 0.0007s | Model Forward Time: 2.5830s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0007 | max_forward: 0.1685 | max_loss: 0.0137 | max_backward: 0.0851 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.2906 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 5532.6 | uss_MB: 4487.8 | child_rss_MB: 9818.0 | allocated_MB: 609.0 | reserved_MB: 29572.0 | inactive_split_MB: 268.4 +rank3 [2026-10-27 18:41:12,499] INFO : Step 37000/100000 rank3 | Train Loss: 16.1273 | Train Loss Unweighted: 4.7522 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4049 | Regression_cos_phi: 0.0495 | Regression_energy: 0.5975 | Regression_eta: 0.0881 | Regression_pt: 0.5494 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6829 | physical_nopu: 4.5107 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4533 | Classification_binary: 3.4623 | Regression_cos_phi: 2.1856 | Regression_energy: 4.0065 | Regression_eta: 1.2436 | Regression_pt: 4.8866 | Regression_sin_phi: 1.8585 | LR: 2.91e-04 | DataLoad Time: 0.0029s | Model Forward Time: 2.3161s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0884 | max_loss: 0.0036 | max_backward: 0.0963 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 1.0635 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0656 | Memory: rss_MB: 5529.2 | uss_MB: 4574.6 | child_rss_MB: 9790.1 | allocated_MB: 605.6 | reserved_MB: 29572.0 | inactive_split_MB: 233.2 +rank3 [2026-10-27 18:44:44,282] INFO : Step 38000/100000 rank3 | Train Loss: 12.5142 | Train Loss Unweighted: 4.4218 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.6290 | Regression_cos_phi: 0.0475 | Regression_energy: 0.3376 | Regression_eta: 0.0660 | Regression_pt: 0.2827 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6875 | physical_nopu: 4.5136 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3131 | Classification_binary: 3.6053 | Regression_cos_phi: 1.7076 | Regression_energy: 1.6020 | Regression_eta: 1.1499 | Regression_pt: 1.7971 | Regression_sin_phi: 1.6128 | LR: 2.86e-04 | DataLoad Time: 0.0012s | Model Forward Time: 2.4331s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.0807 | max_loss: 0.0039 | max_backward: 0.1583 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.2386 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5519.2 | uss_MB: 4574.3 | child_rss_MB: 9713.9 | allocated_MB: 607.7 | reserved_MB: 29572.0 | inactive_split_MB: 203.1 +rank3 [2026-10-27 18:48:30,202] INFO : Step 39000/100000 rank3 | Train Loss: 13.9834 | Train Loss Unweighted: 4.4509 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4391 | Regression_cos_phi: 0.0477 | Regression_energy: 0.4363 | Regression_eta: 0.0792 | Regression_pt: 0.3839 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.6802 | physical_nopu: 4.5058 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4179 | Classification_binary: 3.3473 | Regression_cos_phi: 1.6684 | Regression_energy: 1.5795 | Regression_eta: 0.9378 | Regression_pt: 1.9203 | Regression_sin_phi: 1.4891 | LR: 2.81e-04 | DataLoad Time: 0.0027s | Model Forward Time: 3.8719s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.0950 | max_loss: 0.0039 | max_backward: 0.1102 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 2.6970 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5506.6 | uss_MB: 4574.6 | child_rss_MB: 9623.2 | allocated_MB: 611.4 | reserved_MB: 29682.0 | inactive_split_MB: 276.6 +rank3 [2026-10-27 18:52:07,824] INFO : Step 40000/100000 rank3 | Train Loss: 15.4467 | Train Loss Unweighted: 4.8241 | Train Loss Components: Classification: 0.0121 | Classification_binary: 3.5538 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5661 | Regression_eta: 0.0744 | Regression_pt: 0.5256 | Regression_sin_phi: 0.0462 | Group Train Loss Unweighted (window mean): physical_pu: 3.6754 | physical_nopu: 4.5045 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3875 | Classification_binary: 3.6975 | Regression_cos_phi: 1.6349 | Regression_energy: 1.7245 | Regression_eta: 1.0088 | Regression_pt: 1.9309 | Regression_sin_phi: 1.9129 | LR: 2.76e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.4736s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.1251 | max_loss: 0.0037 | max_backward: 0.0913 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2799 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5491.6 | uss_MB: 4574.5 | child_rss_MB: 9510.6 | allocated_MB: 607.6 | reserved_MB: 29682.0 | inactive_split_MB: 251.4 +rank3 [2026-10-27 18:52:08,682] INFO : Running validation on rank3 +rank3 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 18:54:11,846] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 18:54:12,297] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 18:54:37,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 18:54:37,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 18:54:38,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 18:54:38,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 18:54:39,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 18:54:39,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 18:54:40,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 18:54:40,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 18:54:50,967] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-27 18:54:51,317] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 18:54:51,717] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 18:55:06,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 18:55:07,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 18:55:07,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 18:55:08,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 18:55:08,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 18:55:09,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 18:55:09,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 18:55:10,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 18:55:20,331] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 18:55:21,167] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 18:55:21,566] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 18:55:36,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 18:55:37,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 18:55:37,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 18:55:38,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 18:55:38,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 18:55:39,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 18:55:39,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 18:55:40,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 18:55:50,472] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 19:00:08,492] INFO : Step 41000/100000 rank3 | Train Loss: 13.2101 | Train Loss Unweighted: 4.4466 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5620 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3777 | Regression_eta: 0.0720 | Regression_pt: 0.3238 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6774 | physical_nopu: 4.4925 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4234 | Classification_binary: 3.4761 | Regression_cos_phi: 1.8562 | Regression_energy: 1.8213 | Regression_eta: 1.0537 | Regression_pt: 1.9468 | Regression_sin_phi: 2.0953 | LR: 2.70e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.5580s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0904 | max_forward: 0.2227 | max_loss: 0.0038 | max_backward: 0.1239 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 2.1580 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6748.4 | uss_MB: 5831.9 | child_rss_MB: 9514.0 | allocated_MB: 608.3 | reserved_MB: 27230.0 | inactive_split_MB: 387.2 +rank3 [2026-10-27 19:03:45,853] INFO : Step 42000/100000 rank3 | Train Loss: 13.7061 | Train Loss Unweighted: 4.4760 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4108 | Regression_cos_phi: 0.0459 | Regression_energy: 0.4702 | Regression_eta: 0.0614 | Regression_pt: 0.4341 | Regression_sin_phi: 0.0374 | Group Train Loss Unweighted (window mean): physical_pu: 3.6711 | physical_nopu: 4.4967 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3686 | Classification_binary: 3.3544 | Regression_cos_phi: 1.5903 | Regression_energy: 2.2472 | Regression_eta: 0.6303 | Regression_pt: 2.6242 | Regression_sin_phi: 1.4653 | LR: 2.65e-04 | DataLoad Time: 0.0067s | Model Forward Time: 2.9860s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0008 | max_forward: 0.1920 | max_loss: 0.0038 | max_backward: 0.1427 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 1.6213 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6739.4 | uss_MB: 5628.8 | child_rss_MB: 9436.4 | allocated_MB: 609.4 | reserved_MB: 27632.0 | inactive_split_MB: 509.5 +rank3 [2026-10-27 19:07:23,511] INFO : Step 43000/100000 rank3 | Train Loss: 11.9489 | Train Loss Unweighted: 3.6938 | Train Loss Components: Classification: 0.0162 | Classification_binary: 2.9682 | Regression_cos_phi: 0.0472 | Regression_energy: 0.2764 | Regression_eta: 0.1089 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6702 | physical_nopu: 4.4944 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4954 | Classification_binary: 2.9978 | Regression_cos_phi: 1.8942 | Regression_energy: 1.4966 | Regression_eta: 1.8025 | Regression_pt: 1.6511 | Regression_sin_phi: 2.0505 | LR: 2.59e-04 | DataLoad Time: 0.0015s | Model Forward Time: 3.8332s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0032 | max_forward: 0.2195 | max_loss: 0.0114 | max_backward: 0.4513 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 2.0889 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0865 | Memory: rss_MB: 5560.5 | uss_MB: 3524.0 | child_rss_MB: 9348.2 | allocated_MB: 619.9 | reserved_MB: 27918.0 | inactive_split_MB: 1305.5 +rank3 [2026-10-27 19:11:14,851] INFO : Step 44000/100000 rank3 | Train Loss: 15.8790 | Train Loss Unweighted: 4.7893 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4610 | Regression_cos_phi: 0.0438 | Regression_energy: 0.5944 | Regression_eta: 0.0747 | Regression_pt: 0.5488 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6681 | physical_nopu: 4.4856 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5322 | Classification_binary: 3.4732 | Regression_cos_phi: 1.9944 | Regression_energy: 3.7691 | Regression_eta: 1.2809 | Regression_pt: 5.4815 | Regression_sin_phi: 1.7258 | LR: 2.54e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.1651s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0980 | max_loss: 0.0531 | max_backward: 0.1371 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.9403 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5553.5 | uss_MB: 4666.8 | child_rss_MB: 9287.7 | allocated_MB: 606.4 | reserved_MB: 31154.0 | inactive_split_MB: 295.0 +rank3 [2026-10-27 19:15:01,382] INFO : Step 45000/100000 rank3 | Train Loss: 13.8786 | Train Loss Unweighted: 4.5241 | Train Loss Components: Classification: 0.0100 | Classification_binary: 3.5822 | Regression_cos_phi: 0.0536 | Regression_energy: 0.4068 | Regression_eta: 0.0847 | Regression_pt: 0.3344 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 3.6667 | physical_nopu: 4.4836 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2655 | Classification_binary: 3.5177 | Regression_cos_phi: 1.4651 | Regression_energy: 1.5167 | Regression_eta: 0.6490 | Regression_pt: 1.8335 | Regression_sin_phi: 1.4336 | LR: 2.48e-04 | DataLoad Time: 0.0011s | Model Forward Time: 4.6044s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0009 | max_forward: 0.0951 | max_loss: 0.0815 | max_backward: 0.1837 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 3.3466 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 5551.4 | uss_MB: 4666.0 | child_rss_MB: 9284.1 | allocated_MB: 608.8 | reserved_MB: 31154.0 | inactive_split_MB: 289.8 +rank3 [2026-10-27 19:18:42,715] INFO : Step 46000/100000 rank3 | Train Loss: 13.1587 | Train Loss Unweighted: 4.3779 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.4817 | Regression_cos_phi: 0.0459 | Regression_energy: 0.3805 | Regression_eta: 0.0668 | Regression_pt: 0.3375 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.6620 | physical_nopu: 4.4831 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6308 | Classification_binary: 3.4970 | Regression_cos_phi: 1.3690 | Regression_energy: 2.0037 | Regression_eta: 1.0960 | Regression_pt: 2.3691 | Regression_sin_phi: 1.6317 | LR: 2.43e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.3332s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0924 | max_forward: 0.1910 | max_loss: 0.0039 | max_backward: 0.1028 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 2.0151 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5545.4 | uss_MB: 4666.2 | child_rss_MB: 9248.1 | allocated_MB: 609.2 | reserved_MB: 31154.0 | inactive_split_MB: 277.4 +rank3 [2026-10-27 19:22:31,522] INFO : Step 47000/100000 rank3 | Train Loss: 15.2118 | Train Loss Unweighted: 4.6831 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3624 | Regression_cos_phi: 0.0409 | Regression_energy: 0.5987 | Regression_eta: 0.0659 | Regression_pt: 0.5598 | Regression_sin_phi: 0.0405 | Group Train Loss Unweighted (window mean): physical_pu: 3.6609 | physical_nopu: 4.4795 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5296 | Classification_binary: 3.1116 | Regression_cos_phi: 1.4985 | Regression_energy: 3.4506 | Regression_eta: 0.7934 | Regression_pt: 4.4525 | Regression_sin_phi: 1.1975 | LR: 2.37e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.2741s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0777 | max_forward: 0.1864 | max_loss: 0.0077 | max_backward: 0.1105 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 1.8392 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5533.2 | uss_MB: 4665.8 | child_rss_MB: 9129.7 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 305.0 +rank3 [2026-10-27 19:26:21,057] INFO : Step 48000/100000 rank3 | Train Loss: 12.0455 | Train Loss Unweighted: 4.3014 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.6150 | Regression_cos_phi: 0.0526 | Regression_energy: 0.2805 | Regression_eta: 0.0652 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6617 | physical_nopu: 4.4726 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3472 | Classification_binary: 3.7075 | Regression_cos_phi: 2.2419 | Regression_energy: 1.7991 | Regression_eta: 1.0300 | Regression_pt: 1.8710 | Regression_sin_phi: 2.0983 | LR: 2.31e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.3650s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0531 | max_forward: 0.1607 | max_loss: 0.0044 | max_backward: 0.0997 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 2.0345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5534.2 | uss_MB: 4664.3 | child_rss_MB: 9129.2 | allocated_MB: 606.8 | reserved_MB: 31154.0 | inactive_split_MB: 262.3 +rank3 [2026-10-27 19:30:06,213] INFO : Step 49000/100000 rank3 | Train Loss: 13.3973 | Train Loss Unweighted: 4.4035 | Train Loss Components: Classification: 0.0168 | Classification_binary: 3.5033 | Regression_cos_phi: 0.0506 | Regression_energy: 0.3797 | Regression_eta: 0.0752 | Regression_pt: 0.3299 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6566 | physical_nopu: 4.4735 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4938 | Classification_binary: 3.3853 | Regression_cos_phi: 2.1945 | Regression_energy: 2.1890 | Regression_eta: 1.0058 | Regression_pt: 2.3670 | Regression_sin_phi: 2.0058 | LR: 2.26e-04 | DataLoad Time: 0.0022s | Model Forward Time: 4.5484s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0220 | max_forward: 0.1075 | max_loss: 0.0037 | max_backward: 0.1020 | max_optimizer_step: 0.2722 | max_tensorboard_logging: 3.2606 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5532.9 | uss_MB: 4389.0 | child_rss_MB: 9133.5 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 312.3 +rank3 [2026-10-27 19:33:47,825] INFO : Step 50000/100000 rank3 | Train Loss: 14.2027 | Train Loss Unweighted: 4.5299 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4272 | Regression_cos_phi: 0.0432 | Regression_energy: 0.4877 | Regression_eta: 0.0670 | Regression_pt: 0.4429 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4710 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5751 | Classification_binary: 3.3559 | Regression_cos_phi: 1.7402 | Regression_energy: 3.3777 | Regression_eta: 1.0754 | Regression_pt: 4.2493 | Regression_sin_phi: 1.5987 | LR: 2.20e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.0498s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0964 | max_loss: 0.0590 | max_backward: 0.1520 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.8031 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5519.9 | uss_MB: 4665.8 | child_rss_MB: 9015.1 | allocated_MB: 607.2 | reserved_MB: 31154.0 | inactive_split_MB: 255.6 +rank3 [2026-10-27 19:33:48,728] INFO : Running validation on rank3 +rank3 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 19:35:46,023] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 19:35:46,495] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 19:36:12,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 19:36:12,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 19:36:13,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 19:36:13,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 19:36:14,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 19:36:14,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 19:36:15,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 19:36:15,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 19:36:25,790] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 19:36:25,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 19:36:26,397] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 19:36:41,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 19:36:42,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 19:36:42,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 19:36:43,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 19:36:43,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 19:36:44,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 19:36:44,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 19:36:45,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 19:36:55,244] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 19:36:55,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 19:36:55,958] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 19:37:11,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 19:37:11,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 19:37:12,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 19:37:12,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 19:37:13,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 19:37:13,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 19:37:14,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 19:37:14,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 19:37:24,607] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 19:41:35,791] INFO : Step 51000/100000 rank3 | Train Loss: 13.0997 | Train Loss Unweighted: 4.3733 | Train Loss Components: Classification: 0.0129 | Classification_binary: 3.5282 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3598 | Regression_eta: 0.0785 | Regression_pt: 0.2920 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6545 | physical_nopu: 4.4631 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3796 | Classification_binary: 3.4610 | Regression_cos_phi: 3.0889 | Regression_energy: 3.0000 | Regression_eta: 3.0574 | Regression_pt: 2.9116 | Regression_sin_phi: 2.8062 | LR: 2.14e-04 | DataLoad Time: 0.0111s | Model Forward Time: 3.1043s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1313 | max_loss: 0.0271 | max_backward: 0.1689 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.7590 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6858.3 | uss_MB: 6017.9 | child_rss_MB: 8958.7 | allocated_MB: 607.6 | reserved_MB: 29536.0 | inactive_split_MB: 388.7 +rank3 [2026-10-27 19:45:15,993] INFO : Step 52000/100000 rank3 | Train Loss: 11.7496 | Train Loss Unweighted: 3.6574 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9475 | Regression_cos_phi: 0.0469 | Regression_energy: 0.2717 | Regression_eta: 0.1056 | Regression_pt: 0.2231 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6567 | physical_nopu: 4.4685 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4705 | Classification_binary: 2.9409 | Regression_cos_phi: 1.8124 | Regression_energy: 1.4367 | Regression_eta: 1.6097 | Regression_pt: 1.5844 | Regression_sin_phi: 1.8385 | LR: 2.09e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.6142s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.1059 | max_forward: 0.2956 | max_loss: 0.0038 | max_backward: 0.4187 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 1.8177 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6858.3 | uss_MB: 6026.1 | child_rss_MB: 8942.2 | allocated_MB: 619.9 | reserved_MB: 29990.0 | inactive_split_MB: 1078.6 +rank3 [2026-10-27 19:48:51,528] INFO : Step 53000/100000 rank3 | Train Loss: 12.5294 | Train Loss Unweighted: 4.3083 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5302 | Regression_cos_phi: 0.0494 | Regression_energy: 0.3242 | Regression_eta: 0.0722 | Regression_pt: 0.2705 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6530 | physical_nopu: 4.4629 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4264 | Classification_binary: 3.5156 | Regression_cos_phi: 1.8662 | Regression_energy: 1.9131 | Regression_eta: 1.0394 | Regression_pt: 2.1159 | Regression_sin_phi: 1.6868 | LR: 2.03e-04 | DataLoad Time: 0.0010s | Model Forward Time: 2.5900s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.1740 | max_loss: 0.0036 | max_backward: 0.1135 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 1.3124 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 5606.5 | uss_MB: 4766.5 | child_rss_MB: 8943.1 | allocated_MB: 608.5 | reserved_MB: 29990.0 | inactive_split_MB: 246.4 +rank3 [2026-10-27 19:52:28,507] INFO : Step 54000/100000 rank3 | Train Loss: 13.6142 | Train Loss Unweighted: 4.3355 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.3742 | Regression_cos_phi: 0.0499 | Regression_energy: 0.4083 | Regression_eta: 0.0736 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4652 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4695 | Classification_binary: 3.3773 | Regression_cos_phi: 1.6204 | Regression_energy: 2.8906 | Regression_eta: 0.7517 | Regression_pt: 3.5863 | Regression_sin_phi: 1.7282 | LR: 1.97e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.4442s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1029 | max_loss: 0.0156 | max_backward: 0.1263 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2459 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 5600.5 | uss_MB: 4766.6 | child_rss_MB: 8886.2 | allocated_MB: 608.0 | reserved_MB: 29990.0 | inactive_split_MB: 249.1 +rank3 [2026-10-27 19:56:02,481] INFO : Step 55000/100000 rank3 | Train Loss: 14.4463 | Train Loss Unweighted: 4.6356 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5250 | Regression_cos_phi: 0.0497 | Regression_energy: 0.4959 | Regression_eta: 0.0660 | Regression_pt: 0.4403 | Regression_sin_phi: 0.0455 | Group Train Loss Unweighted (window mean): physical_pu: 3.6517 | physical_nopu: 4.4554 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3456 | Classification_binary: 3.4980 | Regression_cos_phi: 2.1255 | Regression_energy: 2.5642 | Regression_eta: 0.8396 | Regression_pt: 2.7736 | Regression_sin_phi: 2.0513 | LR: 1.92e-04 | DataLoad Time: 0.0010s | Model Forward Time: 2.9814s | Phase Times: max_pre_step_barrier: 0.0108 | max_batch_to_device: 0.0008 | max_forward: 0.0923 | max_loss: 0.0219 | max_backward: 0.1306 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 1.7796 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5599.8 | uss_MB: 4765.5 | child_rss_MB: 8883.4 | allocated_MB: 610.0 | reserved_MB: 29990.0 | inactive_split_MB: 274.0 +rank3 [2026-10-27 19:59:40,797] INFO : Step 56000/100000 rank3 | Train Loss: 12.4091 | Train Loss Unweighted: 4.2572 | Train Loss Components: Classification: 0.0135 | Classification_binary: 3.4523 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3421 | Regression_eta: 0.0664 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6493 | physical_nopu: 4.4547 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4354 | Classification_binary: 3.3870 | Regression_cos_phi: 1.5661 | Regression_energy: 2.1252 | Regression_eta: 1.5477 | Regression_pt: 2.3710 | Regression_sin_phi: 2.5925 | LR: 1.86e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.0680s | Phase Times: max_pre_step_barrier: 0.3290 | max_batch_to_device: 0.0902 | max_forward: 0.1771 | max_loss: 0.0038 | max_backward: 0.1058 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 1.4273 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5599.8 | uss_MB: 4766.9 | child_rss_MB: 8880.9 | allocated_MB: 608.1 | reserved_MB: 29990.0 | inactive_split_MB: 253.9 +rank3 [2026-10-27 20:03:29,496] INFO : Step 57000/100000 rank3 | Train Loss: 14.9087 | Train Loss Unweighted: 4.5957 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4371 | Regression_cos_phi: 0.0471 | Regression_energy: 0.5028 | Regression_eta: 0.0770 | Regression_pt: 0.4658 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6475 | physical_nopu: 4.4509 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5010 | Classification_binary: 3.4588 | Regression_cos_phi: 2.5300 | Regression_energy: 2.3917 | Regression_eta: 1.6256 | Regression_pt: 2.8208 | Regression_sin_phi: 1.6559 | LR: 1.81e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.4959s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1036 | max_loss: 0.0876 | max_backward: 0.2130 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 1.1951 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5597.8 | uss_MB: 4497.3 | child_rss_MB: 8866.5 | allocated_MB: 608.1 | reserved_MB: 30358.0 | inactive_split_MB: 283.9 +rank3 [2026-10-27 20:07:08,735] INFO : Step 58000/100000 rank3 | Train Loss: 12.4468 | Train Loss Unweighted: 4.2826 | Train Loss Components: Classification: 0.0105 | Classification_binary: 3.5299 | Regression_cos_phi: 0.0545 | Regression_energy: 0.3106 | Regression_eta: 0.0620 | Regression_pt: 0.2643 | Regression_sin_phi: 0.0509 | Group Train Loss Unweighted (window mean): physical_pu: 3.6440 | physical_nopu: 4.4492 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3416 | Classification_binary: 3.5948 | Regression_cos_phi: 2.1215 | Regression_energy: 1.6003 | Regression_eta: 0.9997 | Regression_pt: 1.7964 | Regression_sin_phi: 2.5144 | LR: 1.75e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.5432s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1868 | max_loss: 0.0037 | max_backward: 0.1111 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2860 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5587.6 | uss_MB: 3611.2 | child_rss_MB: 8803.1 | allocated_MB: 608.2 | reserved_MB: 30358.0 | inactive_split_MB: 337.0 +rank3 [2026-10-27 20:10:52,442] INFO : Step 59000/100000 rank3 | Train Loss: 13.1870 | Train Loss Unweighted: 4.3659 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4760 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3771 | Regression_eta: 0.0773 | Regression_pt: 0.3247 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4440 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5299 | Classification_binary: 3.6605 | Regression_cos_phi: 1.6445 | Regression_energy: 1.9993 | Regression_eta: 1.0632 | Regression_pt: 2.2640 | Regression_sin_phi: 2.2169 | LR: 1.70e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4658s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1005 | max_loss: 0.0036 | max_backward: 0.1062 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 1.2390 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 5582.6 | uss_MB: 4766.7 | child_rss_MB: 8755.9 | allocated_MB: 610.1 | reserved_MB: 31234.0 | inactive_split_MB: 273.9 +rank3 [2026-10-27 20:14:34,818] INFO : Step 60000/100000 rank3 | Train Loss: 15.4000 | Train Loss Unweighted: 4.6905 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.3905 | Regression_cos_phi: 0.0456 | Regression_energy: 0.5805 | Regression_eta: 0.0703 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6406 | physical_nopu: 4.4523 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3638 | Classification_binary: 3.4454 | Regression_cos_phi: 1.5962 | Regression_energy: 2.5246 | Regression_eta: 1.2077 | Regression_pt: 2.9355 | Regression_sin_phi: 1.5106 | LR: 1.64e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.5442s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0757 | max_forward: 0.1624 | max_loss: 0.0040 | max_backward: 0.1315 | max_optimizer_step: 0.2155 | max_tensorboard_logging: 1.2531 | max_tensorboard_flush: 0.0020 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5582.8 | uss_MB: 4766.6 | child_rss_MB: 8760.5 | allocated_MB: 611.6 | reserved_MB: 31234.0 | inactive_split_MB: 318.4 +rank3 [2026-10-27 20:14:35,735] INFO : Running validation on rank3 +rank3 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 20:16:30,817] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 20:16:31,258] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 20:16:55,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 20:16:55,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 20:16:56,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 20:16:56,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 20:16:57,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 20:16:57,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 20:16:58,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 20:16:58,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 20:17:09,010] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 20:17:09,353] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 20:17:09,706] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 20:17:24,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 20:17:25,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 20:17:25,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 20:17:26,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 20:17:26,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 20:17:27,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 20:17:27,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 20:17:28,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 20:17:38,216] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 20:17:39,021] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 20:17:39,450] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 20:17:54,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 20:17:55,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 20:17:55,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 20:17:56,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 20:17:56,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 20:17:57,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 20:17:57,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 20:17:58,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 20:18:08,217] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 20:22:28,768] INFO : Step 61000/100000 rank3 | Train Loss: 12.1474 | Train Loss Unweighted: 4.3289 | Train Loss Components: Classification: 0.0110 | Classification_binary: 3.6236 | Regression_cos_phi: 0.0523 | Regression_energy: 0.2909 | Regression_eta: 0.0654 | Regression_pt: 0.2368 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.6358 | physical_nopu: 4.4439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3063 | Classification_binary: 3.8406 | Regression_cos_phi: 1.9196 | Regression_energy: 1.4100 | Regression_eta: 1.3961 | Regression_pt: 1.5925 | Regression_sin_phi: 1.9219 | LR: 1.59e-04 | DataLoad Time: 0.0077s | Model Forward Time: 3.4729s | Phase Times: max_pre_step_barrier: 0.0115 | max_batch_to_device: 0.0009 | max_forward: 0.2092 | max_loss: 0.0037 | max_backward: 0.1501 | max_optimizer_step: 0.2965 | max_tensorboard_logging: 1.8725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6817.2 | uss_MB: 6009.9 | child_rss_MB: 8755.0 | allocated_MB: 607.8 | reserved_MB: 28822.0 | inactive_split_MB: 222.9 +rank3 [2026-10-27 20:26:14,158] INFO : Step 62000/100000 rank3 | Train Loss: 11.7977 | Train Loss Unweighted: 3.6294 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9084 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2778 | Regression_eta: 0.1039 | Regression_pt: 0.2286 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6417 | physical_nopu: 4.4459 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5156 | Classification_binary: 2.8551 | Regression_cos_phi: 1.8761 | Regression_energy: 1.5366 | Regression_eta: 1.5892 | Regression_pt: 1.7053 | Regression_sin_phi: 1.9738 | LR: 1.54e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.7424s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0033 | max_forward: 0.2219 | max_loss: 0.0037 | max_backward: 0.2965 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1292 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1278 | Memory: rss_MB: 6812.4 | uss_MB: 6017.7 | child_rss_MB: 8760.3 | allocated_MB: 618.8 | reserved_MB: 28822.0 | inactive_split_MB: 888.0 +rank3 [2026-10-27 20:30:05,877] INFO : Step 63000/100000 rank3 | Train Loss: 12.5071 | Train Loss Unweighted: 4.3613 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5997 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3200 | Regression_eta: 0.0719 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6399 | physical_nopu: 4.4447 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4216 | Classification_binary: 3.4080 | Regression_cos_phi: 2.1206 | Regression_energy: 1.4683 | Regression_eta: 0.9295 | Regression_pt: 1.7852 | Regression_sin_phi: 1.6784 | LR: 1.49e-04 | DataLoad Time: 0.0024s | Model Forward Time: 11.0657s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1169 | max_loss: 0.0157 | max_backward: 0.2882 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 9.0944 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6806.2 | uss_MB: 4694.0 | child_rss_MB: 8747.4 | allocated_MB: 610.4 | reserved_MB: 28822.0 | inactive_split_MB: 261.6 +rank3 [2026-10-27 20:33:53,442] INFO : Step 64000/100000 rank3 | Train Loss: 14.0388 | Train Loss Unweighted: 4.5035 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.4200 | Regression_cos_phi: 0.0430 | Regression_energy: 0.4752 | Regression_eta: 0.0663 | Regression_pt: 0.4376 | Regression_sin_phi: 0.0442 | Group Train Loss Unweighted (window mean): physical_pu: 3.6357 | physical_nopu: 4.4395 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5257 | Classification_binary: 3.6153 | Regression_cos_phi: 1.6773 | Regression_energy: 1.6671 | Regression_eta: 1.1189 | Regression_pt: 1.9015 | Regression_sin_phi: 2.2364 | LR: 1.43e-04 | DataLoad Time: 0.0025s | Model Forward Time: 6.4865s | Phase Times: max_pre_step_barrier: 0.1731 | max_batch_to_device: 0.0118 | max_forward: 0.1219 | max_loss: 0.0038 | max_backward: 0.1121 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 5.0441 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5600.0 | uss_MB: 4820.3 | child_rss_MB: 8744.3 | allocated_MB: 611.1 | reserved_MB: 29052.0 | inactive_split_MB: 372.9 +rank3 [2026-10-27 20:37:48,515] INFO : Step 65000/100000 rank3 | Train Loss: 16.3374 | Train Loss Unweighted: 4.8919 | Train Loss Components: Classification: 0.0142 | Classification_binary: 3.4871 | Regression_cos_phi: 0.0500 | Regression_energy: 0.6360 | Regression_eta: 0.0628 | Regression_pt: 0.5943 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4335 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4427 | Classification_binary: 3.5111 | Regression_cos_phi: 1.5190 | Regression_energy: 2.5543 | Regression_eta: 0.9511 | Regression_pt: 3.2711 | Regression_sin_phi: 1.8940 | LR: 1.38e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.4886s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.2167 | max_loss: 0.0058 | max_backward: 0.2407 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 1.9060 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5594.0 | uss_MB: 4820.8 | child_rss_MB: 8750.8 | allocated_MB: 609.7 | reserved_MB: 29052.0 | inactive_split_MB: 236.9 +rank3 [2026-10-27 20:41:37,389] INFO : Step 66000/100000 rank3 | Train Loss: 12.8848 | Train Loss Unweighted: 4.2935 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4715 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3451 | Regression_eta: 0.0693 | Regression_pt: 0.2907 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.6349 | physical_nopu: 4.4348 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4301 | Classification_binary: 3.3297 | Regression_cos_phi: 1.7491 | Regression_energy: 1.4764 | Regression_eta: 0.7895 | Regression_pt: 1.6456 | Regression_sin_phi: 1.4820 | LR: 1.33e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.1925s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0854 | max_loss: 0.1083 | max_backward: 0.2178 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.9218 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5582.1 | uss_MB: 4820.7 | child_rss_MB: 8750.8 | allocated_MB: 611.4 | reserved_MB: 29052.0 | inactive_split_MB: 254.2 +rank3 [2026-10-27 20:45:24,536] INFO : Step 67000/100000 rank3 | Train Loss: 14.2125 | Train Loss Unweighted: 4.5026 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.4493 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4566 | Regression_eta: 0.0671 | Regression_pt: 0.4150 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6352 | physical_nopu: 4.4350 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4288 | Classification_binary: 3.6858 | Regression_cos_phi: 2.2310 | Regression_energy: 2.4800 | Regression_eta: 1.1864 | Regression_pt: 2.8683 | Regression_sin_phi: 1.8184 | LR: 1.28e-04 | DataLoad Time: 0.0027s | Model Forward Time: 3.0116s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0901 | max_loss: 0.0487 | max_backward: 0.1449 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 1.8090 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5575.1 | uss_MB: 4820.8 | child_rss_MB: 8750.2 | allocated_MB: 607.1 | reserved_MB: 29260.0 | inactive_split_MB: 246.6 +rank3 [2026-10-27 20:49:05,126] INFO : Step 68000/100000 rank3 | Train Loss: 12.0487 | Train Loss Unweighted: 4.2633 | Train Loss Components: Classification: 0.0107 | Classification_binary: 3.5135 | Regression_cos_phi: 0.0466 | Regression_energy: 0.3165 | Regression_eta: 0.0645 | Regression_pt: 0.2647 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4387 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3394 | Classification_binary: 3.7149 | Regression_cos_phi: 1.5427 | Regression_energy: 1.5582 | Regression_eta: 0.8824 | Regression_pt: 1.7225 | Regression_sin_phi: 1.6558 | LR: 1.24e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.8889s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0010 | max_forward: 0.0879 | max_loss: 0.0120 | max_backward: 0.1069 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.7176 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5570.9 | uss_MB: 4820.5 | child_rss_MB: 8753.3 | allocated_MB: 609.5 | reserved_MB: 29260.0 | inactive_split_MB: 225.9 +rank3 [2026-10-27 20:52:47,283] INFO : Step 69000/100000 rank3 | Train Loss: 13.1377 | Train Loss Unweighted: 4.3133 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4496 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3632 | Regression_eta: 0.0778 | Regression_pt: 0.3079 | Regression_sin_phi: 0.0541 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4321 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4960 | Classification_binary: 3.5133 | Regression_cos_phi: 1.5279 | Regression_energy: 1.7950 | Regression_eta: 1.4582 | Regression_pt: 1.9985 | Regression_sin_phi: 2.4384 | LR: 1.19e-04 | DataLoad Time: 0.0016s | Model Forward Time: 4.9126s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.1207 | max_loss: 0.0509 | max_backward: 0.2012 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 3.5532 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 5571.9 | uss_MB: 4820.8 | child_rss_MB: 8755.8 | allocated_MB: 608.0 | reserved_MB: 29260.0 | inactive_split_MB: 256.4 +rank3 [2026-10-27 20:56:27,513] INFO : Step 70000/100000 rank3 | Train Loss: 15.8477 | Train Loss Unweighted: 4.7317 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3873 | Regression_cos_phi: 0.0498 | Regression_energy: 0.6015 | Regression_eta: 0.0629 | Regression_pt: 0.5683 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6315 | physical_nopu: 4.4280 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6226 | Classification_binary: 3.4014 | Regression_cos_phi: 1.8585 | Regression_energy: 4.9581 | Regression_eta: 1.0808 | Regression_pt: 6.2605 | Regression_sin_phi: 1.5733 | LR: 1.14e-04 | DataLoad Time: 0.0057s | Model Forward Time: 2.9381s | Phase Times: max_pre_step_barrier: 0.0198 | max_batch_to_device: 0.0007 | max_forward: 0.1178 | max_loss: 0.0036 | max_backward: 0.1081 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.7610 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5571.5 | uss_MB: 4819.7 | child_rss_MB: 8751.0 | allocated_MB: 610.3 | reserved_MB: 30150.0 | inactive_split_MB: 260.3 +rank3 [2026-10-27 20:56:28,457] INFO : Running validation on rank3 +rank3 [2026-10-27 20:56:28,457] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 20:56:28,457] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 20:56:28,457] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 20:58:25,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 20:58:26,360] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 20:58:51,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 20:58:52,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 20:58:53,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 20:58:53,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 20:58:54,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 20:58:54,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 20:58:55,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 20:58:55,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 20:59:05,669] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-27 20:59:05,946] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 20:59:06,379] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 20:59:21,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 20:59:22,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 20:59:22,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 20:59:23,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 20:59:23,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 20:59:24,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 20:59:24,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 20:59:25,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 20:59:35,323] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 20:59:35,862] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 20:59:36,279] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 20:59:51,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 20:59:51,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 20:59:52,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 20:59:52,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 20:59:53,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 20:59:53,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 20:59:54,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 20:59:54,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 21:00:04,841] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 21:04:25,115] INFO : Step 71000/100000 rank3 | Train Loss: 12.0110 | Train Loss Unweighted: 3.6539 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9046 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2881 | Regression_eta: 0.1063 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6306 | physical_nopu: 4.4256 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4844 | Classification_binary: 2.8890 | Regression_cos_phi: 1.7434 | Regression_energy: 1.4631 | Regression_eta: 1.6919 | Regression_pt: 1.5824 | Regression_sin_phi: 1.8534 | LR: 1.10e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.4584s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0035 | max_forward: 0.1952 | max_loss: 0.0038 | max_backward: 0.3195 | max_optimizer_step: 0.3092 | max_tensorboard_logging: 1.7411 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6798.7 | uss_MB: 6058.3 | child_rss_MB: 8722.6 | allocated_MB: 623.5 | reserved_MB: 27422.0 | inactive_split_MB: 864.2 +rank3 [2026-10-27 21:08:01,993] INFO : Step 72000/100000 rank3 | Train Loss: 15.2878 | Train Loss Unweighted: 4.6447 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.3643 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5735 | Regression_eta: 0.0647 | Regression_pt: 0.5353 | Regression_sin_phi: 0.0449 | Group Train Loss Unweighted (window mean): physical_pu: 3.6275 | physical_nopu: 4.4239 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5363 | Classification_binary: 3.3862 | Regression_cos_phi: 1.4767 | Regression_energy: 3.5612 | Regression_eta: 0.9642 | Regression_pt: 4.5201 | Regression_sin_phi: 1.4733 | LR: 1.05e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.5451s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0232 | max_forward: 0.1174 | max_loss: 0.0036 | max_backward: 0.0921 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 1.2606 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5641.1 | uss_MB: 4896.7 | child_rss_MB: 8682.2 | allocated_MB: 607.6 | reserved_MB: 28586.0 | inactive_split_MB: 303.9 +rank3 [2026-10-27 21:11:50,556] INFO : Step 73000/100000 rank3 | Train Loss: 12.7380 | Train Loss Unweighted: 4.2392 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.4842 | Regression_cos_phi: 0.0575 | Regression_energy: 0.3093 | Regression_eta: 0.0643 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.6309 | physical_nopu: 4.4270 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3790 | Classification_binary: 3.3391 | Regression_cos_phi: 2.5490 | Regression_energy: 1.8716 | Regression_eta: 1.2471 | Regression_pt: 2.1082 | Regression_sin_phi: 2.2280 | LR: 1.01e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.4474s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0985 | max_loss: 0.0632 | max_backward: 0.1654 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2139 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5637.1 | uss_MB: 4897.0 | child_rss_MB: 8681.3 | allocated_MB: 607.0 | reserved_MB: 28586.0 | inactive_split_MB: 227.5 +rank3 [2026-10-27 21:15:34,267] INFO : Step 74000/100000 rank3 | Train Loss: 12.5735 | Train Loss Unweighted: 4.2478 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4076 | Regression_cos_phi: 0.0429 | Regression_energy: 0.3533 | Regression_eta: 0.0699 | Regression_pt: 0.3116 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6307 | physical_nopu: 4.4218 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4492 | Classification_binary: 3.3158 | Regression_cos_phi: 1.4999 | Regression_energy: 1.8354 | Regression_eta: 0.9345 | Regression_pt: 2.0802 | Regression_sin_phi: 2.0046 | LR: 9.68e-05 | DataLoad Time: 0.0023s | Model Forward Time: 2.5461s | Phase Times: max_pre_step_barrier: 0.1687 | max_batch_to_device: 0.0008 | max_forward: 0.0990 | max_loss: 0.0516 | max_backward: 0.1766 | max_optimizer_step: 0.1812 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5626.2 | uss_MB: 4897.3 | child_rss_MB: 8634.5 | allocated_MB: 608.6 | reserved_MB: 28586.0 | inactive_split_MB: 285.4 +rank3 [2026-10-27 21:19:21,651] INFO : Step 75000/100000 rank3 | Train Loss: 15.1687 | Train Loss Unweighted: 4.6098 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.3751 | Regression_cos_phi: 0.0492 | Regression_energy: 0.5530 | Regression_eta: 0.0638 | Regression_pt: 0.5060 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4206 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3477 | Classification_binary: 3.5574 | Regression_cos_phi: 1.7995 | Regression_energy: 1.9535 | Regression_eta: 1.0372 | Regression_pt: 2.0393 | Regression_sin_phi: 2.0653 | LR: 9.27e-05 | DataLoad Time: 0.0025s | Model Forward Time: 2.4831s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1026 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.2820 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5619.2 | uss_MB: 4896.7 | child_rss_MB: 8602.7 | allocated_MB: 610.1 | reserved_MB: 28586.0 | inactive_split_MB: 259.7 +rank3 [2026-10-27 21:23:05,981] INFO : Step 76000/100000 rank3 | Train Loss: 12.2943 | Train Loss Unweighted: 4.2773 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5085 | Regression_cos_phi: 0.0476 | Regression_energy: 0.3262 | Regression_eta: 0.0732 | Regression_pt: 0.2631 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6289 | physical_nopu: 4.4209 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4146 | Classification_binary: 3.5379 | Regression_cos_phi: 1.8510 | Regression_energy: 2.2380 | Regression_eta: 1.1327 | Regression_pt: 2.1171 | Regression_sin_phi: 1.9968 | LR: 8.88e-05 | DataLoad Time: 0.0020s | Model Forward Time: 2.6228s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.1085 | max_optimizer_step: 0.1831 | max_tensorboard_logging: 1.4262 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 5617.3 | uss_MB: 4895.6 | child_rss_MB: 8601.6 | allocated_MB: 608.9 | reserved_MB: 28942.0 | inactive_split_MB: 227.1 +rank3 [2026-10-27 21:27:01,963] INFO : Step 77000/100000 rank3 | Train Loss: 13.8477 | Train Loss Unweighted: 4.4496 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4225 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4510 | Regression_eta: 0.0623 | Regression_pt: 0.4032 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6274 | physical_nopu: 4.4166 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5250 | Classification_binary: 3.4829 | Regression_cos_phi: 1.5976 | Regression_energy: 2.5656 | Regression_eta: 0.7993 | Regression_pt: 3.1564 | Regression_sin_phi: 2.0486 | LR: 8.50e-05 | DataLoad Time: 0.0010s | Model Forward Time: 6.1313s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0839 | max_forward: 0.1754 | max_loss: 0.0036 | max_backward: 0.1103 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 4.6772 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1374 | Memory: rss_MB: 5607.3 | uss_MB: 4838.5 | child_rss_MB: 8502.5 | allocated_MB: 611.0 | reserved_MB: 28942.0 | inactive_split_MB: 369.0 +rank3 [2026-10-27 21:30:52,663] INFO : Step 78000/100000 rank3 | Train Loss: 14.1981 | Train Loss Unweighted: 4.6212 | Train Loss Components: Classification: 0.0133 | Classification_binary: 3.5747 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4573 | Regression_eta: 0.0669 | Regression_pt: 0.4114 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.6241 | physical_nopu: 4.4230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4358 | Classification_binary: 3.4779 | Regression_cos_phi: 2.0775 | Regression_energy: 2.9774 | Regression_eta: 0.7241 | Regression_pt: 3.7977 | Regression_sin_phi: 1.7032 | LR: 8.13e-05 | DataLoad Time: 0.0021s | Model Forward Time: 3.0985s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1149 | max_loss: 0.0636 | max_backward: 0.1914 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5607.3 | uss_MB: 4897.1 | child_rss_MB: 8495.1 | allocated_MB: 606.6 | reserved_MB: 28942.0 | inactive_split_MB: 309.2 +rank3 [2026-10-27 21:34:49,720] INFO : Step 79000/100000 rank3 | Train Loss: 12.3648 | Train Loss Unweighted: 4.2396 | Train Loss Components: Classification: 0.0145 | Classification_binary: 3.4636 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3226 | Regression_eta: 0.0628 | Regression_pt: 0.2792 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6254 | physical_nopu: 4.4183 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4901 | Classification_binary: 3.4238 | Regression_cos_phi: 1.6553 | Regression_energy: 1.3831 | Regression_eta: 1.0182 | Regression_pt: 1.5774 | Regression_sin_phi: 1.6853 | LR: 7.78e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.3387s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0936 | max_forward: 0.1936 | max_loss: 0.0037 | max_backward: 0.1134 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 2.0079 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 5602.4 | uss_MB: 4897.2 | child_rss_MB: 8483.1 | allocated_MB: 609.2 | reserved_MB: 29222.0 | inactive_split_MB: 324.6 +rank3 [2026-10-27 21:38:37,320] INFO : Step 80000/100000 rank3 | Train Loss: 16.1833 | Train Loss Unweighted: 4.8046 | Train Loss Components: Classification: 0.0155 | Classification_binary: 3.4149 | Regression_cos_phi: 0.0453 | Regression_energy: 0.6189 | Regression_eta: 0.0801 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6233 | physical_nopu: 4.4142 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5119 | Classification_binary: 3.3186 | Regression_cos_phi: 1.4090 | Regression_energy: 3.4107 | Regression_eta: 1.2812 | Regression_pt: 4.0109 | Regression_sin_phi: 2.1390 | LR: 7.44e-05 | DataLoad Time: 0.0103s | Model Forward Time: 3.9234s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0008 | max_forward: 0.1071 | max_loss: 0.0037 | max_backward: 0.1380 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 2.6784 | max_tensorboard_flush: 0.0026 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 5596.4 | uss_MB: 4871.2 | child_rss_MB: 8436.2 | allocated_MB: 606.3 | reserved_MB: 29222.0 | inactive_split_MB: 471.7 +rank3 [2026-10-27 21:38:38,322] INFO : Running validation on rank3 +rank3 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 21:40:39,510] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 21:40:39,982] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 21:41:05,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 21:41:06,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 21:41:06,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 21:41:07,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 21:41:07,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 21:41:08,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 21:41:08,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 21:41:09,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 21:41:19,240] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 21:41:19,914] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 21:41:20,296] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 21:41:35,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 21:41:36,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 21:41:36,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 21:41:37,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 21:41:37,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 21:41:38,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 21:41:38,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 21:41:39,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 21:41:49,275] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-27 21:41:49,545] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 21:41:50,005] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 21:42:04,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 21:42:05,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 21:42:05,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 21:42:06,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 21:42:06,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 21:42:07,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 21:42:07,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 21:42:08,354] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 21:42:18,365] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 21:46:50,145] INFO : Step 81000/100000 rank3 | Train Loss: 11.7500 | Train Loss Unweighted: 3.6065 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.8934 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2712 | Regression_eta: 0.1074 | Regression_pt: 0.2244 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4130 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4838 | Classification_binary: 2.8218 | Regression_cos_phi: 1.6713 | Regression_energy: 1.4610 | Regression_eta: 1.6946 | Regression_pt: 1.6136 | Regression_sin_phi: 1.8484 | LR: 7.11e-05 | DataLoad Time: 0.0020s | Model Forward Time: 8.6906s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0975 | max_forward: 0.3237 | max_loss: 0.0036 | max_backward: 0.5496 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 6.3197 | max_tensorboard_flush: 0.0011 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6963.5 | uss_MB: 4835.8 | child_rss_MB: 8415.8 | allocated_MB: 619.9 | reserved_MB: 28598.0 | inactive_split_MB: 1192.4 +rank3 [2026-10-27 21:50:33,876] INFO : Step 82000/100000 rank3 | Train Loss: 13.6704 | Train Loss Unweighted: 4.5604 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0419 | Regression_energy: 0.4618 | Regression_eta: 0.0599 | Regression_pt: 0.4291 | Regression_sin_phi: 0.0407 | Group Train Loss Unweighted (window mean): physical_pu: 3.6250 | physical_nopu: 4.4152 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4988 | Classification_binary: 3.4078 | Regression_cos_phi: 1.1548 | Regression_energy: 2.3457 | Regression_eta: 0.7815 | Regression_pt: 2.9762 | Regression_sin_phi: 1.3694 | LR: 6.80e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.1350s | Phase Times: max_pre_step_barrier: 0.1835 | max_batch_to_device: 0.0874 | max_forward: 0.1976 | max_loss: 0.0036 | max_backward: 0.1170 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 1.6260 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 5659.6 | uss_MB: 4976.9 | child_rss_MB: 8356.3 | allocated_MB: 607.5 | reserved_MB: 28598.0 | inactive_split_MB: 253.8 +rank3 [2026-10-27 21:54:19,227] INFO : Step 83000/100000 rank3 | Train Loss: 12.0270 | Train Loss Unweighted: 4.2414 | Train Loss Components: Classification: 0.0092 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0499 | Regression_energy: 0.3056 | Regression_eta: 0.0671 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4141 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2998 | Classification_binary: 3.6652 | Regression_cos_phi: 2.2643 | Regression_energy: 1.2908 | Regression_eta: 1.0796 | Regression_pt: 1.4124 | Regression_sin_phi: 1.8230 | LR: 6.51e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.7063s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1966 | max_loss: 0.0138 | max_backward: 0.1124 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 2.4026 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 5659.6 | uss_MB: 4977.1 | child_rss_MB: 8350.2 | allocated_MB: 609.6 | reserved_MB: 28598.0 | inactive_split_MB: 289.5 +rank3 [2026-10-27 21:58:08,515] INFO : Step 84000/100000 rank3 | Train Loss: 12.5172 | Train Loss Unweighted: 4.2412 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.4173 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3464 | Regression_eta: 0.0637 | Regression_pt: 0.3052 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4149 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5266 | Classification_binary: 3.5681 | Regression_cos_phi: 2.0671 | Regression_energy: 2.2173 | Regression_eta: 1.3950 | Regression_pt: 2.4936 | Regression_sin_phi: 2.5033 | LR: 6.23e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.2648s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1103 | max_loss: 0.0037 | max_backward: 0.1159 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 2.0319 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5655.6 | uss_MB: 4977.8 | child_rss_MB: 8312.0 | allocated_MB: 605.9 | reserved_MB: 28598.0 | inactive_split_MB: 208.8 +rank3 [2026-10-27 22:01:46,561] INFO : Step 85000/100000 rank3 | Train Loss: 15.2814 | Train Loss Unweighted: 4.6863 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.3811 | Regression_cos_phi: 0.0432 | Regression_energy: 0.5842 | Regression_eta: 0.0608 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6173 | physical_nopu: 4.4078 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4740 | Classification_binary: 3.3348 | Regression_cos_phi: 1.3955 | Regression_energy: 3.4443 | Regression_eta: 0.8353 | Regression_pt: 4.2487 | Regression_sin_phi: 1.6615 | LR: 5.96e-05 | DataLoad Time: 0.0021s | Model Forward Time: 3.0837s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.0993 | max_loss: 0.0558 | max_backward: 0.1762 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 1.8022 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 5645.3 | uss_MB: 4977.4 | child_rss_MB: 8242.1 | allocated_MB: 606.6 | reserved_MB: 28598.0 | inactive_split_MB: 257.4 +rank3 [2026-10-27 22:05:28,665] INFO : Step 86000/100000 rank3 | Train Loss: 12.8074 | Train Loss Unweighted: 4.3373 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.5785 | Regression_cos_phi: 0.0543 | Regression_energy: 0.3116 | Regression_eta: 0.0747 | Regression_pt: 0.2515 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.6255 | physical_nopu: 4.4086 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3500 | Classification_binary: 3.5302 | Regression_cos_phi: 2.6524 | Regression_energy: 1.4965 | Regression_eta: 0.9865 | Regression_pt: 1.4761 | Regression_sin_phi: 2.1593 | LR: 5.71e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.0435s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1215 | max_loss: 0.0037 | max_backward: 0.1229 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 1.6761 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5636.3 | uss_MB: 4977.9 | child_rss_MB: 8180.8 | allocated_MB: 608.3 | reserved_MB: 29688.0 | inactive_split_MB: 271.8 +rank3 [2026-10-27 22:09:22,620] INFO : Step 87000/100000 rank3 | Train Loss: 12.9093 | Train Loss Unweighted: 4.3003 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.3807 | Regression_cos_phi: 0.0447 | Regression_energy: 0.4014 | Regression_eta: 0.0615 | Regression_pt: 0.3523 | Regression_sin_phi: 0.0424 | Group Train Loss Unweighted (window mean): physical_pu: 3.6211 | physical_nopu: 4.4120 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5596 | Classification_binary: 3.3181 | Regression_cos_phi: 1.2064 | Regression_energy: 1.7534 | Regression_eta: 0.8386 | Regression_pt: 2.0554 | Regression_sin_phi: 1.5583 | LR: 5.48e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.0853s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0203 | max_forward: 0.1299 | max_loss: 0.0038 | max_backward: 0.1175 | max_optimizer_step: 0.2192 | max_tensorboard_logging: 1.8010 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5636.1 | uss_MB: 4575.9 | child_rss_MB: 8182.7 | allocated_MB: 607.0 | reserved_MB: 29688.0 | inactive_split_MB: 231.0 +rank3 [2026-10-27 22:13:02,799] INFO : Step 88000/100000 rank3 | Train Loss: 15.2369 | Train Loss Unweighted: 4.6456 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4525 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5240 | Regression_eta: 0.0665 | Regression_pt: 0.4840 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6192 | physical_nopu: 4.4119 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5180 | Classification_binary: 3.3520 | Regression_cos_phi: 1.9565 | Regression_energy: 3.4684 | Regression_eta: 0.9799 | Regression_pt: 4.4279 | Regression_sin_phi: 3.2410 | LR: 5.26e-05 | DataLoad Time: 0.0009s | Model Forward Time: 3.1181s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0007 | max_forward: 0.1127 | max_loss: 0.0236 | max_backward: 0.1542 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 1.7956 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5636.1 | uss_MB: 4977.3 | child_rss_MB: 8181.7 | allocated_MB: 609.0 | reserved_MB: 29688.0 | inactive_split_MB: 221.0 +rank3 [2026-10-27 22:16:52,919] INFO : Step 89000/100000 rank3 | Train Loss: 12.8609 | Train Loss Unweighted: 4.2765 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.4665 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3343 | Regression_eta: 0.0681 | Regression_pt: 0.2896 | Regression_sin_phi: 0.0555 | Group Train Loss Unweighted (window mean): physical_pu: 3.6177 | physical_nopu: 4.4110 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4085 | Classification_binary: 3.3711 | Regression_cos_phi: 1.5604 | Regression_energy: 1.8971 | Regression_eta: 0.9395 | Regression_pt: 2.1025 | Regression_sin_phi: 2.1476 | LR: 5.06e-05 | DataLoad Time: 0.0027s | Model Forward Time: 3.1951s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1194 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 1.9452 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 5632.5 | uss_MB: 4977.7 | child_rss_MB: 8137.7 | allocated_MB: 612.6 | reserved_MB: 30628.0 | inactive_split_MB: 309.7 +rank3 [2026-10-27 22:20:43,724] INFO : Step 90000/100000 rank3 | Train Loss: 11.6102 | Train Loss Unweighted: 3.6127 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9177 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2661 | Regression_eta: 0.1040 | Regression_pt: 0.2155 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4109 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4416 | Classification_binary: 2.8695 | Regression_cos_phi: 1.7556 | Regression_energy: 1.3681 | Regression_eta: 1.5796 | Regression_pt: 1.4716 | Regression_sin_phi: 1.9305 | LR: 4.88e-05 | DataLoad Time: 0.0021s | Model Forward Time: 3.5772s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0110 | max_forward: 0.1856 | max_loss: 0.0145 | max_backward: 0.4526 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 1.8620 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 5624.5 | uss_MB: 4985.7 | child_rss_MB: 8062.4 | allocated_MB: 617.2 | reserved_MB: 30628.0 | inactive_split_MB: 962.2 +rank3 [2026-10-27 22:20:44,689] INFO : Running validation on rank3 +rank3 [2026-10-27 22:20:44,689] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 22:22:45,204] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 22:22:45,606] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 22:23:10,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 22:23:10,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 22:23:11,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 22:23:11,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 22:23:12,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 22:23:12,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 22:23:13,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 22:23:13,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 22:23:23,995] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 22:23:24,365] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 22:23:24,733] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 22:23:39,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 22:23:40,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 22:23:41,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 22:23:41,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 22:23:42,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 22:23:42,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 22:23:43,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 22:23:43,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 22:23:53,620] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 22:23:53,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 22:23:54,273] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 22:24:09,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 22:24:09,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 22:24:10,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 22:24:10,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 22:24:11,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 22:24:11,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 22:24:12,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 22:24:12,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 22:24:22,630] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 22:28:41,211] INFO : Step 91000/100000 rank3 | Train Loss: 12.9063 | Train Loss Unweighted: 4.3371 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5236 | Regression_cos_phi: 0.0531 | Regression_energy: 0.3431 | Regression_eta: 0.0707 | Regression_pt: 0.2829 | Regression_sin_phi: 0.0512 | Group Train Loss Unweighted (window mean): physical_pu: 3.6180 | physical_nopu: 4.4087 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4058 | Classification_binary: 3.5946 | Regression_cos_phi: 2.1107 | Regression_energy: 1.8432 | Regression_eta: 1.3579 | Regression_pt: 1.8741 | Regression_sin_phi: 2.3637 | LR: 4.71e-05 | DataLoad Time: 0.0018s | Model Forward Time: 4.9001s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0055 | max_forward: 0.1521 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 3.4850 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7066.6 | uss_MB: 6432.9 | child_rss_MB: 8020.4 | allocated_MB: 607.8 | reserved_MB: 28796.0 | inactive_split_MB: 320.2 +rank3 [2026-10-27 22:32:23,431] INFO : Step 92000/100000 rank3 | Train Loss: 15.4341 | Train Loss Unweighted: 4.7247 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4285 | Regression_cos_phi: 0.0457 | Regression_energy: 0.5811 | Regression_eta: 0.0735 | Regression_pt: 0.5375 | Regression_sin_phi: 0.0425 | Group Train Loss Unweighted (window mean): physical_pu: 3.6150 | physical_nopu: 4.4032 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5955 | Classification_binary: 3.3939 | Regression_cos_phi: 1.5929 | Regression_energy: 3.4871 | Regression_eta: 0.9882 | Regression_pt: 4.5630 | Regression_sin_phi: 1.5372 | LR: 4.57e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.3947s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.2696 | max_loss: 0.0037 | max_backward: 0.1715 | max_optimizer_step: 0.3237 | max_tensorboard_logging: 1.7768 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7051.6 | uss_MB: 6432.9 | child_rss_MB: 7931.4 | allocated_MB: 609.3 | reserved_MB: 28796.0 | inactive_split_MB: 222.8 +rank3 [2026-10-27 22:36:08,796] INFO : Step 93000/100000 rank3 | Train Loss: 13.9744 | Train Loss Unweighted: 4.4854 | Train Loss Components: Classification: 0.0118 | Classification_binary: 3.5124 | Regression_cos_phi: 0.0578 | Regression_energy: 0.4245 | Regression_eta: 0.0652 | Regression_pt: 0.3624 | Regression_sin_phi: 0.0513 | Group Train Loss Unweighted (window mean): physical_pu: 3.6205 | physical_nopu: 4.4075 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3578 | Classification_binary: 3.4186 | Regression_cos_phi: 3.5388 | Regression_energy: 4.1330 | Regression_eta: 0.8128 | Regression_pt: 5.2156 | Regression_sin_phi: 2.8752 | LR: 4.43e-05 | DataLoad Time: 0.0024s | Model Forward Time: 2.9846s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1098 | max_loss: 0.0036 | max_backward: 0.1173 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 1.7709 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5706.9 | uss_MB: 5092.9 | child_rss_MB: 7894.0 | allocated_MB: 606.4 | reserved_MB: 30394.0 | inactive_split_MB: 244.2 +rank3 [2026-10-27 22:39:53,036] INFO : Step 94000/100000 rank3 | Train Loss: 12.4397 | Train Loss Unweighted: 4.2538 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4230 | Regression_cos_phi: 0.0415 | Regression_energy: 0.3513 | Regression_eta: 0.0702 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6164 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4273 | Classification_binary: 3.5583 | Regression_cos_phi: 1.4875 | Regression_energy: 1.8857 | Regression_eta: 1.4591 | Regression_pt: 2.3000 | Regression_sin_phi: 2.1356 | LR: 4.32e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.1513s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0009 | max_forward: 0.1540 | max_loss: 0.0036 | max_backward: 0.2309 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 1.7723 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 5695.9 | uss_MB: 4753.0 | child_rss_MB: 7806.9 | allocated_MB: 607.1 | reserved_MB: 30394.0 | inactive_split_MB: 232.2 +rank3 [2026-10-27 22:43:36,954] INFO : Step 95000/100000 rank3 | Train Loss: 14.9177 | Train Loss Unweighted: 4.6108 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3911 | Regression_cos_phi: 0.0436 | Regression_energy: 0.5612 | Regression_eta: 0.0598 | Regression_pt: 0.4905 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6121 | physical_nopu: 4.4076 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5482 | Classification_binary: 3.2755 | Regression_cos_phi: 1.3043 | Regression_energy: 6.9653 | Regression_eta: 1.1666 | Regression_pt: 7.3380 | Regression_sin_phi: 3.7841 | LR: 4.22e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.0352s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1111 | max_loss: 0.0199 | max_backward: 0.1395 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.7978 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5695.9 | uss_MB: 5093.1 | child_rss_MB: 7811.5 | allocated_MB: 609.0 | reserved_MB: 30394.0 | inactive_split_MB: 285.3 +rank3 [2026-10-27 22:47:14,355] INFO : Step 96000/100000 rank3 | Train Loss: 12.4738 | Train Loss Unweighted: 4.2244 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.4884 | Regression_cos_phi: 0.0540 | Regression_energy: 0.2967 | Regression_eta: 0.0804 | Regression_pt: 0.2429 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6137 | physical_nopu: 4.4081 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2794 | Classification_binary: 3.4008 | Regression_cos_phi: 2.5152 | Regression_energy: 1.4278 | Regression_eta: 1.0151 | Regression_pt: 1.4987 | Regression_sin_phi: 1.7603 | LR: 4.14e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.1604s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0007 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.2003 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 1.8675 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5695.9 | uss_MB: 5093.2 | child_rss_MB: 7804.9 | allocated_MB: 608.1 | reserved_MB: 30394.0 | inactive_split_MB: 259.4 +rank3 [2026-10-27 22:50:52,983] INFO : Step 97000/100000 rank3 | Train Loss: 12.7922 | Train Loss Unweighted: 4.2852 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.4355 | Regression_cos_phi: 0.0455 | Regression_energy: 0.3604 | Regression_eta: 0.0703 | Regression_pt: 0.3091 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6136 | physical_nopu: 4.4015 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5920 | Classification_binary: 3.2788 | Regression_cos_phi: 1.8441 | Regression_energy: 2.0872 | Regression_eta: 1.1520 | Regression_pt: 2.3863 | Regression_sin_phi: 1.9525 | LR: 4.08e-05 | DataLoad Time: 0.0015s | Model Forward Time: 3.0113s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1110 | max_loss: 0.0174 | max_backward: 0.1239 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 1.7282 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0908 | Memory: rss_MB: 5687.4 | uss_MB: 5092.4 | child_rss_MB: 7746.9 | allocated_MB: 607.2 | reserved_MB: 30394.0 | inactive_split_MB: 265.3 +rank3 [2026-10-27 22:54:49,599] INFO : Step 98000/100000 rank3 | Train Loss: 13.8508 | Train Loss Unweighted: 4.4512 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3736 | Regression_cos_phi: 0.0443 | Regression_energy: 0.4792 | Regression_eta: 0.0606 | Regression_pt: 0.4354 | Regression_sin_phi: 0.0433 | Group Train Loss Unweighted (window mean): physical_pu: 3.6169 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4566 | Classification_binary: 3.3498 | Regression_cos_phi: 1.9300 | Regression_energy: 2.8472 | Regression_eta: 0.9836 | Regression_pt: 3.4722 | Regression_sin_phi: 2.1274 | LR: 4.04e-05 | DataLoad Time: 0.0025s | Model Forward Time: 3.0652s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0119 | max_forward: 0.1317 | max_loss: 0.0036 | max_backward: 0.1246 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.7312 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0878 | Memory: rss_MB: 5687.4 | uss_MB: 5093.6 | child_rss_MB: 7731.7 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 292.7 +rank3 [2026-10-27 22:58:30,099] INFO : Step 99000/100000 rank3 | Train Loss: 12.5729 | Train Loss Unweighted: 4.3128 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5536 | Regression_cos_phi: 0.0519 | Regression_energy: 0.3097 | Regression_eta: 0.0644 | Regression_pt: 0.2679 | Regression_sin_phi: 0.0529 | Group Train Loss Unweighted (window mean): physical_pu: 3.6154 | physical_nopu: 4.4035 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4224 | Classification_binary: 3.3196 | Regression_cos_phi: 1.7760 | Regression_energy: 1.8647 | Regression_eta: 0.9964 | Regression_pt: 2.2244 | Regression_sin_phi: 1.9921 | LR: 4.01e-05 | DataLoad Time: 0.0102s | Model Forward Time: 3.0453s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0008 | max_forward: 0.1158 | max_loss: 0.0113 | max_backward: 0.1168 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 1.7726 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 5687.4 | uss_MB: 5093.5 | child_rss_MB: 7727.1 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 283.7 +rank3 [2026-10-27 23:02:17,099] INFO : Step 100000/100000 rank3 | Train Loss: 11.9488 | Train Loss Unweighted: 3.6450 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9057 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2836 | Regression_eta: 0.1038 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6201 | physical_nopu: 4.4070 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5064 | Classification_binary: 2.8735 | Regression_cos_phi: 1.7798 | Regression_energy: 1.4690 | Regression_eta: 1.7609 | Regression_pt: 1.6060 | Regression_sin_phi: 1.9645 | LR: 4.00e-05 | DataLoad Time: 0.0031s | Model Forward Time: 6.3803s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0031 | max_forward: 0.2860 | max_loss: 0.0036 | max_backward: 0.4897 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 4.4601 | max_tensorboard_flush: 0.0009 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5687.2 | uss_MB: 4556.1 | child_rss_MB: 7711.3 | allocated_MB: 618.6 | reserved_MB: 30394.0 | inactive_split_MB: 1112.1 +rank3 [2026-10-27 23:02:18,075] INFO : Running validation on rank3 +rank3 [2026-10-27 23:02:18,075] INFO : ResumableSampler physical_pu:valid resetting +rank3 [2026-10-27 23:02:18,075] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank3 [2026-10-27 23:02:18,075] INFO : ResumableSampler physical_nopu:valid resetting +rank3 [2026-10-27 23:02:18,075] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank3 [2026-10-27 23:04:23,624] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 23:04:24,087] INFO : Running predictions on cms_pf_ttbar +rank3 [2026-10-27 23:04:49,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_3_0.parquet +rank3 [2026-10-27 23:04:50,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_3_1.parquet +rank3 [2026-10-27 23:04:50,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_3_2.parquet +rank3 [2026-10-27 23:04:51,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_3_3.parquet +rank3 [2026-10-27 23:04:51,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_3_4.parquet +rank3 [2026-10-27 23:04:52,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_3_5.parquet +rank3 [2026-10-27 23:04:52,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_3_6.parquet +rank3 [2026-10-27 23:04:53,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_3_7.parquet +rank3 [2026-10-27 23:05:03,228] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 23:05:03,811] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 23:05:04,178] INFO : Running predictions on cms_pf_qcd +rank3 [2026-10-27 23:05:19,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_3_0.parquet +rank3 [2026-10-27 23:05:20,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_3_1.parquet +rank3 [2026-10-27 23:05:20,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_3_2.parquet +rank3 [2026-10-27 23:05:21,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_3_3.parquet +rank3 [2026-10-27 23:05:21,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_3_4.parquet +rank3 [2026-10-27 23:05:22,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_3_5.parquet +rank3 [2026-10-27 23:05:22,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_3_6.parquet +rank3 [2026-10-27 23:05:23,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_3_7.parquet +rank3 [2026-10-27 23:05:33,232] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 23:05:33,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-27 23:05:34,200] INFO : Running predictions on cms_pf_ztt +rank3 [2026-10-27 23:05:49,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_3_0.parquet +rank3 [2026-10-27 23:05:49,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_3_1.parquet +rank3 [2026-10-27 23:05:50,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_3_2.parquet +rank3 [2026-10-27 23:05:50,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_3_3.parquet +rank3 [2026-10-27 23:05:51,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_3_4.parquet +rank3 [2026-10-27 23:05:51,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_3_5.parquet +rank3 [2026-10-27 23:05:52,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_3_6.parquet +rank3 [2026-10-27 23:05:52,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_3_7.parquet +rank3 [2026-10-27 23:06:02,807] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-27 23:06:03,941] INFO : Training completed. Total time on device 3: 414.789min diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.4 b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.4 new file mode 100644 index 0000000000000000000000000000000000000000..cfc2829f0a54b1a2ac0b4d41a8ef4f8a1e6392ac --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.4 @@ -0,0 +1,897 @@ +rank4 [2026-10-27 16:10:10,397] INFO : Initializing model with seed=12345; process seed=12349 +rank4 [2026-10-27 16:10:10,398] INFO : configured dtype=torch.bfloat16 for autocast +rank4 [2026-10-27 16:10:35,267] INFO : Instantiating model +rank4 [2026-10-27 16:10:35,267] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank4 [2026-10-27 16:10:35,267] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank4 [2026-10-27 16:10:35,296] INFO : Input encoding initialization took 0.03s +rank4 [2026-10-27 16:10:35,296] INFO : nn0 parameters: 2130944 +rank4 [2026-10-27 16:10:35,296] INFO : Initializing shared backbone layers, num_convs=3 +rank4 [2026-10-27 16:10:35,296] INFO : Initializing backbone layer 0 +rank4 [2026-10-27 16:10:35,350] INFO : Initializing backbone layer 1 +rank4 [2026-10-27 16:10:35,380] INFO : Initializing backbone layer 2 +rank4 [2026-10-27 16:10:35,409] INFO : Backbone initialization took 0.11s +rank4 [2026-10-27 16:10:35,410] INFO : backbone parameters: 15765696 +rank4 [2026-10-27 16:10:35,410] INFO : Initializing output DNNs +rank4 [2026-10-27 16:10:35,444] INFO : Output DNNs initialization took 0.03s +rank4 [2026-10-27 16:10:35,444] INFO : backbone_mode=BackboneMode.SHARED +rank4 [2026-10-27 16:10:35,444] INFO : jagged_attention enabled=False +rank4 [2026-10-27 16:10:35,444] INFO : flash_attn_varlen enabled=False +rank4 [2026-10-27 16:10:35,444] INFO : task_queries enabled=False +rank4 [2026-10-27 16:10:35,444] INFO : classification_norm parameters: 0 +rank4 [2026-10-27 16:10:35,444] INFO : regression_norm parameters: 0 +rank4 [2026-10-27 16:10:35,444] INFO : classification_query parameters: 0 +rank4 [2026-10-27 16:10:35,444] INFO : regression_query parameters: 0 +rank4 [2026-10-27 16:10:35,444] INFO : classification_readout parameters: 0 +rank4 [2026-10-27 16:10:35,444] INFO : regression_readout parameters: 0 +rank4 [2026-10-27 16:10:35,444] INFO : output_mode=elementwise +rank4 [2026-10-27 16:10:35,445] INFO : set_decoder parameters: 0 +rank4 [2026-10-27 16:10:35,445] INFO : nn_binary_particle parameters: 264706 +rank4 [2026-10-27 16:10:35,445] INFO : nn_pid parameters: 268297 +rank4 [2026-10-27 16:10:35,445] INFO : nn_pt parameters: 2113544 +rank4 [2026-10-27 16:10:35,445] INFO : nn_eta parameters: 264706 +rank4 [2026-10-27 16:10:35,445] INFO : nn_sin_phi parameters: 264706 +rank4 [2026-10-27 16:10:35,445] INFO : nn_cos_phi parameters: 264706 +rank4 [2026-10-27 16:10:35,445] INFO : nn_energy parameters: 2113544 +rank4 [2026-10-27 16:10:35,446] INFO : Total MLPF parameters: 23450849 +rank4 [2026-10-27 16:10:35,446] INFO : MLPF __init__ done +rank4 [2026-10-27 16:10:35,446] INFO : Instantiated model +rank4 [2026-10-27 16:10:35,446] INFO : Moving model to device rank=4 +rank4 [2026-10-27 16:10:35,479] INFO : Moved model to device rank=4 +rank4 [2026-10-27 16:10:35,480] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank4 [2026-10-27 16:10:35,481] INFO : Configured model for SyncBatchNorm rank=4 +rank4 [2026-10-27 16:10:36,005] INFO : Configured model for DistributedDataParallel rank=4 +rank4 [2026-10-27 16:10:36,125] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +202 module.nn_energy.nn.7.0.bias ... 0 +203 module.nn_energy.nn.7.2.weight ... 0 +204 module.nn_energy.nn.7.2.bias ... 0 +205 module.nn_energy.nn.7.4.weight ... 0 +206 module.nn_energy.nn.7.4.bias ... 0 + +[207 rows x 3 columns] +rank4 [2026-10-27 16:10:36,125] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x HEPTv2Layer( + embedding_dim=512, num_heads=16, width=512, dropout=0.0, block_size=100, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=512, out_features=512, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=512, out_features=512, bias=False) + (w_k): Linear(in_features=512, out_features=512, bias=False) + (w_v): Linear(in_features=512, out_features=512, bias=False) + (w_rpe): Linear(in_features=10, out_features=512, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=2048, bias=True) + (1): LayerNorm((2048,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=2048, out_features=512, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=512, out_features=2048, bias=False) + (up_proj): Linear(in_features=512, out_features=2048, bias=False) + (down_proj): Linear(in_features=2048, out_features=512, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank4 [2026-10-27 16:10:36,126] INFO : Trainable parameters: 23450849 +rank4 [2026-10-27 16:10:36,126] INFO : Non-trainable parameters: 22386 +rank4 [2026-10-27 16:10:36,126] INFO : Total parameters: 23473235 +rank4 [2026-10-27 16:10:36,128] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 262144 0 + module.backbone.0.attn.out_linear.bias 512 0 + module.backbone.0.attn.e2lsh.alpha 0 1632 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 262144 0 + module.backbone.0.w_k.weight 262144 0 + module.backbone.0.w_v.weight 262144 0 + module.backbone.0.w_rpe.weight 0 5120 + module.backbone.0.w_rpe.bias 0 512 +module.backbone.0.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm2.weight 512 0 + module.backbone.0.q_norm.weight 32 0 + module.backbone.0.k_norm.weight 32 0 + module.backbone.0.ff.gate_proj.weight 1048576 0 + module.backbone.0.ff.up_proj.weight 1048576 0 + module.backbone.0.ff.down_proj.weight 1048576 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 262144 0 + module.backbone.1.attn.out_linear.bias 512 0 + module.backbone.1.attn.e2lsh.alpha 0 1632 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 262144 0 + module.backbone.1.w_k.weight 262144 0 + module.backbone.1.w_v.weight 262144 0 + module.backbone.1.w_rpe.weight 0 5120 + module.backbone.1.w_rpe.bias 0 512 +module.backbone.1.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm2.weight 512 0 + module.backbone.1.q_norm.weight 32 0 + module.backbone.1.k_norm.weight 32 0 + module.backbone.1.ff.gate_proj.weight 1048576 0 + module.backbone.1.ff.up_proj.weight 1048576 0 + module.backbone.1.ff.down_proj.weight 1048576 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 262144 0 + module.backbone.2.attn.out_linear.bias 512 0 + module.backbone.2.attn.e2lsh.alpha 0 1632 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 262144 0 + module.backbone.2.w_k.weight 262144 0 + module.backbone.2.w_v.weight 262144 0 + module.backbone.2.w_rpe.weight 0 5120 + module.backbone.2.w_rpe.bias 0 512 +module.backbone.2.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm2.weight 512 0 + module.backbone.2.q_norm.weight 32 0 + module.backbone.2.k_norm.weight 32 0 + module.backbone.2.ff.gate_proj.weight 1048576 0 + module.backbone.2.ff.up_proj.weight 1048576 0 + module.backbone.2.ff.down_proj.weight 1048576 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank4 [2026-10-27 16:10:36,128] INFO : Getting dataloaders +rank4 [2026-10-27 16:10:36,129] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:43,327] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:44,220] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:44,829] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank4 [2026-10-27 16:10:44,829] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:46,032] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:46,992] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:47,858] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank4 [2026-10-27 16:10:47,858] INFO : Creating WeightedMixIterator with 2 data loaders. +rank4 [2026-10-27 16:10:47,858] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:48,016] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:48,166] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:48,313] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank4 [2026-10-27 16:10:48,313] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:48,476] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:48,643] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:10:48,795] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank4 [2026-10-27 16:10:48,795] INFO : Creating InterleavedIterator with 2 data loaders. +rank4 [2026-10-27 16:11:16,325] INFO : Got dataloaders +rank4 [2026-10-27 16:11:16,325] INFO : loader split=train rank=4 len=232058 +rank4 [2026-10-27 16:11:16,325] INFO : loader split=valid rank=4 len=48 +rank4 [2026-10-27 16:11:16,326] INFO : Starting training from step 1 to 100000 on rank 4 of 8 +rank4 [2026-10-27 16:11:16,326] INFO : Creating train_iterator +rank4 [2026-10-27 16:11:16,326] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank4 [2026-10-27 16:11:30,540] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank4 [2026-10-27 16:11:46,196] INFO : Created train_iterator +rank4 [2026-10-27 16:12:32,254] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.129867712855339, 'Classification': 0.16580097352620216, 'Regression_pt': 0.7168090478330851, 'Regression_eta': 0.335200086296536, 'Regression_sin_phi': 0.1274821354681626, 'Regression_cos_phi': 0.1371471309568733, 'Regression_energy': 0.9554258581250906}, weights={'Classification_binary': 1.0, 'Classification': 30.939912796020508, 'Regression_pt': 7.156533241271973, 'Regression_eta': 15.3038969039917, 'Regression_sin_phi': 40.23989486694336, 'Regression_cos_phi': 37.40412139892578, 'Regression_energy': 5.369194984436035} +rank4 [2026-10-27 16:15:18,322] INFO : Step 1000/100000 rank4 | Train Loss: 16.8632 | Train Loss Unweighted: 5.4934 | Train Loss Components: Classification: 0.0189 | Classification_binary: 4.1833 | Regression_cos_phi: 0.0509 | Regression_energy: 0.5806 | Regression_eta: 0.0776 | Regression_pt: 0.5302 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 5.1301 | physical_nopu: 6.0772 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5064 | Classification_binary: 4.2931 | Regression_cos_phi: 1.6093 | Regression_energy: 2.9075 | Regression_eta: 1.3383 | Regression_pt: 3.5642 | Regression_sin_phi: 2.6033 | LR: 4.00e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.3145s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0014 | max_forward: 0.0387 | max_loss: 0.0036 | max_backward: 0.0664 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 1.2817 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 4374.6 | uss_MB: 3102.3 | child_rss_MB: 11636.1 | allocated_MB: 609.1 | reserved_MB: 27788.0 | inactive_split_MB: 244.1 +rank4 [2026-10-27 16:18:42,007] INFO : Step 2000/100000 rank4 | Train Loss: 17.7204 | Train Loss Unweighted: 5.4242 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.9240 | Regression_cos_phi: 0.0505 | Regression_energy: 0.6758 | Regression_eta: 0.0665 | Regression_pt: 0.6366 | Regression_sin_phi: 0.0551 | Group Train Loss Unweighted (window mean): physical_pu: 4.2802 | physical_nopu: 5.3470 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3634 | Classification_binary: 3.9096 | Regression_cos_phi: 1.8861 | Regression_energy: 3.1085 | Regression_eta: 1.0479 | Regression_pt: 3.5147 | Regression_sin_phi: 1.9127 | LR: 4.00e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.8061s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1422 | max_loss: 0.0036 | max_backward: 0.1717 | max_optimizer_step: 0.1501 | max_tensorboard_logging: 0.7287 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 4367.5 | uss_MB: 3102.8 | child_rss_MB: 11637.6 | allocated_MB: 607.0 | reserved_MB: 27788.0 | inactive_split_MB: 326.9 +rank4 [2026-10-27 16:22:10,507] INFO : Step 3000/100000 rank4 | Train Loss: 14.3102 | Train Loss Unweighted: 4.8875 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.8970 | Regression_cos_phi: 0.0491 | Regression_energy: 0.4277 | Regression_eta: 0.0716 | Regression_pt: 0.3769 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 4.0705 | physical_nopu: 5.1319 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3960 | Classification_binary: 3.9017 | Regression_cos_phi: 2.0671 | Regression_energy: 2.1561 | Regression_eta: 1.0732 | Regression_pt: 2.5401 | Regression_sin_phi: 2.1129 | LR: 3.99e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.0072s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0782 | max_loss: 0.0037 | max_backward: 0.0505 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 1.0271 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4295.2 | uss_MB: 3034.9 | child_rss_MB: 11625.5 | allocated_MB: 607.6 | reserved_MB: 28576.0 | inactive_split_MB: 261.0 +rank4 [2026-10-27 16:25:44,194] INFO : Step 4000/100000 rank4 | Train Loss: 12.6160 | Train Loss Unweighted: 3.9080 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.1145 | Regression_cos_phi: 0.0484 | Regression_energy: 0.3088 | Regression_eta: 0.1117 | Regression_pt: 0.2597 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.9908 | physical_nopu: 5.0219 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5165 | Classification_binary: 3.1155 | Regression_cos_phi: 1.7420 | Regression_energy: 1.5615 | Regression_eta: 1.7413 | Regression_pt: 1.7533 | Regression_sin_phi: 1.8635 | LR: 3.99e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4274s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0032 | max_forward: 0.1365 | max_loss: 0.0036 | max_backward: 0.2496 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 1.0566 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 4292.4 | uss_MB: 3043.1 | child_rss_MB: 11491.5 | allocated_MB: 617.3 | reserved_MB: 28576.0 | inactive_split_MB: 1049.1 +rank4 [2026-10-27 16:29:29,026] INFO : Step 5000/100000 rank4 | Train Loss: 13.7309 | Train Loss Unweighted: 4.7575 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.8762 | Regression_cos_phi: 0.0508 | Regression_energy: 0.3661 | Regression_eta: 0.0922 | Regression_pt: 0.3105 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.9453 | physical_nopu: 4.9561 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4088 | Classification_binary: 3.9400 | Regression_cos_phi: 1.8179 | Regression_energy: 2.2570 | Regression_eta: 1.0903 | Regression_pt: 2.5980 | Regression_sin_phi: 1.7685 | LR: 3.98e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.0871s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.0008 | max_forward: 0.0371 | max_loss: 0.0036 | max_backward: 0.0568 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 2.1174 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4241.6 | uss_MB: 3036.2 | child_rss_MB: 11146.9 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 248.7 +rank4 [2026-10-27 16:33:03,739] INFO : Step 6000/100000 rank4 | Train Loss: 15.2316 | Train Loss Unweighted: 4.9080 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.7479 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5147 | Regression_eta: 0.0679 | Regression_pt: 0.4609 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.9059 | physical_nopu: 4.9043 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6203 | Classification_binary: 3.7123 | Regression_cos_phi: 1.6617 | Regression_energy: 3.4734 | Regression_eta: 1.1348 | Regression_pt: 3.9623 | Regression_sin_phi: 1.6295 | LR: 3.97e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.0045s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0421 | max_loss: 0.0036 | max_backward: 0.0620 | max_optimizer_step: 0.1489 | max_tensorboard_logging: 2.0592 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4216.9 | uss_MB: 3036.3 | child_rss_MB: 11044.6 | allocated_MB: 610.3 | reserved_MB: 28576.0 | inactive_split_MB: 254.0 +rank4 [2026-10-27 16:36:49,903] INFO : Step 7000/100000 rank4 | Train Loss: 14.7215 | Train Loss Unweighted: 4.8917 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8534 | Regression_cos_phi: 0.0503 | Regression_energy: 0.4508 | Regression_eta: 0.0791 | Regression_pt: 0.3921 | Regression_sin_phi: 0.0547 | Group Train Loss Unweighted (window mean): physical_pu: 3.8763 | physical_nopu: 4.8627 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4480 | Classification_binary: 3.9323 | Regression_cos_phi: 1.8599 | Regression_energy: 1.9742 | Regression_eta: 1.0548 | Regression_pt: 2.3047 | Regression_sin_phi: 1.8718 | LR: 3.96e-04 | DataLoad Time: 0.0012s | Model Forward Time: 4.5453s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0348 | max_loss: 0.0036 | max_backward: 0.0535 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 3.5409 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4194.7 | uss_MB: 3033.5 | child_rss_MB: 10983.9 | allocated_MB: 609.0 | reserved_MB: 28576.0 | inactive_split_MB: 216.2 +rank4 [2026-10-27 16:40:34,139] INFO : Step 8000/100000 rank4 | Train Loss: 14.2560 | Train Loss Unweighted: 4.8042 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.8045 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4333 | Regression_eta: 0.0601 | Regression_pt: 0.3883 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.8518 | physical_nopu: 4.8294 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5549 | Classification_binary: 3.9550 | Regression_cos_phi: 2.0416 | Regression_energy: 2.2277 | Regression_eta: 0.8478 | Regression_pt: 2.6896 | Regression_sin_phi: 1.9813 | LR: 3.94e-04 | DataLoad Time: 0.0017s | Model Forward Time: 4.8862s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1342 | max_loss: 0.0036 | max_backward: 0.0500 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 3.8645 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4176.9 | uss_MB: 3036.4 | child_rss_MB: 10934.3 | allocated_MB: 606.6 | reserved_MB: 28576.0 | inactive_split_MB: 255.4 +rank4 [2026-10-27 16:44:13,306] INFO : Step 9000/100000 rank4 | Train Loss: 15.9274 | Train Loss Unweighted: 4.9361 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.6437 | Regression_cos_phi: 0.0472 | Regression_energy: 0.5701 | Regression_eta: 0.0733 | Regression_pt: 0.5360 | Regression_sin_phi: 0.0496 | Group Train Loss Unweighted (window mean): physical_pu: 3.8376 | physical_nopu: 4.8057 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4542 | Classification_binary: 3.6067 | Regression_cos_phi: 1.5160 | Regression_energy: 2.6368 | Regression_eta: 1.1300 | Regression_pt: 3.3199 | Regression_sin_phi: 1.8050 | LR: 3.93e-04 | DataLoad Time: 0.0026s | Model Forward Time: 3.2103s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1320 | max_loss: 0.0037 | max_backward: 0.0555 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 2.0797 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 4157.5 | uss_MB: 3035.9 | child_rss_MB: 10877.2 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 321.4 +rank4 [2026-10-27 16:48:05,432] INFO : Step 10000/100000 rank4 | Train Loss: 13.0514 | Train Loss Unweighted: 4.5682 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.7678 | Regression_cos_phi: 0.0520 | Regression_energy: 0.3372 | Regression_eta: 0.0736 | Regression_pt: 0.2745 | Regression_sin_phi: 0.0521 | Group Train Loss Unweighted (window mean): physical_pu: 3.8237 | physical_nopu: 4.7797 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3545 | Classification_binary: 3.6742 | Regression_cos_phi: 2.1983 | Regression_energy: 1.4088 | Regression_eta: 0.7732 | Regression_pt: 1.6052 | Regression_sin_phi: 2.1310 | LR: 3.91e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.2566s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.0379 | max_loss: 0.0040 | max_backward: 0.0556 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 2.2516 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4149.4 | uss_MB: 3036.1 | child_rss_MB: 10872.4 | allocated_MB: 607.9 | reserved_MB: 28576.0 | inactive_split_MB: 258.7 +rank4 [2026-10-27 16:48:06,334] INFO : Running validation on rank4 +rank4 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 16:50:06,283] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:50:06,771] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 16:50:34,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 16:50:35,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 16:50:36,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 16:50:36,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 16:50:37,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 16:50:37,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 16:50:38,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 16:50:38,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 16:50:48,663] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-27 16:50:48,968] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:50:49,336] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 16:51:04,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 16:51:05,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 16:51:05,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 16:51:06,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 16:51:06,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 16:51:07,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 16:51:07,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 16:51:08,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 16:51:18,151] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 16:51:18,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 16:51:19,153] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 16:51:34,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 16:51:35,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 16:51:35,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 16:51:36,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 16:51:36,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 16:51:37,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 16:51:37,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 16:51:38,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 16:51:48,311] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 16:55:58,693] INFO : Step 11000/100000 rank4 | Train Loss: 14.4095 | Train Loss Unweighted: 4.7539 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.7591 | Regression_cos_phi: 0.0522 | Regression_energy: 0.4228 | Regression_eta: 0.0709 | Regression_pt: 0.3796 | Regression_sin_phi: 0.0518 | Group Train Loss Unweighted (window mean): physical_pu: 3.8077 | physical_nopu: 4.7444 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5615 | Classification_binary: 3.8321 | Regression_cos_phi: 1.9950 | Regression_energy: 2.0759 | Regression_eta: 0.8270 | Regression_pt: 2.5198 | Regression_sin_phi: 1.8789 | LR: 3.89e-04 | DataLoad Time: 0.0007s | Model Forward Time: 3.2010s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1127 | max_loss: 0.0074 | max_backward: 0.1410 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 1.9404 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6580.9 | uss_MB: 5462.7 | child_rss_MB: 10900.1 | allocated_MB: 610.6 | reserved_MB: 27422.0 | inactive_split_MB: 253.4 +rank4 [2026-10-27 16:59:46,314] INFO : Step 12000/100000 rank4 | Train Loss: 16.4725 | Train Loss Unweighted: 5.0863 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.5958 | Regression_cos_phi: 0.0410 | Regression_energy: 0.6842 | Regression_eta: 0.0610 | Regression_pt: 0.6484 | Regression_sin_phi: 0.0394 | Group Train Loss Unweighted (window mean): physical_pu: 3.8013 | physical_nopu: 4.7308 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5661 | Classification_binary: 3.7682 | Regression_cos_phi: 2.2100 | Regression_energy: 3.1581 | Regression_eta: 1.0535 | Regression_pt: 4.0043 | Regression_sin_phi: 1.9443 | LR: 3.87e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.5372s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1292 | max_loss: 0.0037 | max_backward: 0.1951 | max_optimizer_step: 0.2940 | max_tensorboard_logging: 2.1160 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6552.9 | uss_MB: 5462.9 | child_rss_MB: 10800.4 | allocated_MB: 609.5 | reserved_MB: 27422.0 | inactive_split_MB: 347.2 +rank4 [2026-10-27 17:03:40,557] INFO : Step 13000/100000 rank4 | Train Loss: 12.9993 | Train Loss Unweighted: 4.5752 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.7655 | Regression_cos_phi: 0.0483 | Regression_energy: 0.3375 | Regression_eta: 0.0674 | Regression_pt: 0.2918 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7898 | physical_nopu: 4.7063 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3311 | Classification_binary: 3.7472 | Regression_cos_phi: 1.3993 | Regression_energy: 1.9086 | Regression_eta: 0.9959 | Regression_pt: 2.2327 | Regression_sin_phi: 1.7019 | LR: 3.85e-04 | DataLoad Time: 0.0088s | Model Forward Time: 3.0969s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1327 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.9573 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5436.1 | uss_MB: 4383.7 | child_rss_MB: 10499.2 | allocated_MB: 607.2 | reserved_MB: 28584.0 | inactive_split_MB: 340.8 +rank4 [2026-10-27 17:07:23,686] INFO : Step 14000/100000 rank4 | Train Loss: 12.1035 | Train Loss Unweighted: 3.7686 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.0394 | Regression_cos_phi: 0.0492 | Regression_energy: 0.2767 | Regression_eta: 0.1064 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0483 | Group Train Loss Unweighted (window mean): physical_pu: 3.7803 | physical_nopu: 4.6878 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4884 | Classification_binary: 3.1140 | Regression_cos_phi: 1.9063 | Regression_energy: 1.4605 | Regression_eta: 1.6339 | Regression_pt: 1.6143 | Regression_sin_phi: 1.9596 | LR: 3.83e-04 | DataLoad Time: 0.0019s | Model Forward Time: 5.5487s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0034 | max_forward: 0.2002 | max_loss: 0.0036 | max_backward: 0.3668 | max_optimizer_step: 0.2700 | max_tensorboard_logging: 3.9098 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0824 | Memory: rss_MB: 5443.1 | uss_MB: 4393.2 | child_rss_MB: 10497.0 | allocated_MB: 618.5 | reserved_MB: 28584.0 | inactive_split_MB: 893.5 +rank4 [2026-10-27 17:11:07,917] INFO : Step 15000/100000 rank4 | Train Loss: 13.2483 | Train Loss Unweighted: 4.5172 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.6965 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3441 | Regression_eta: 0.0751 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7702 | physical_nopu: 4.6706 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3727 | Classification_binary: 3.6548 | Regression_cos_phi: 2.4940 | Regression_energy: 2.4493 | Regression_eta: 1.4455 | Regression_pt: 2.4305 | Regression_sin_phi: 2.2510 | LR: 3.80e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.3449s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0879 | max_loss: 0.0036 | max_backward: 0.0859 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 2.2540 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0190 | Memory: rss_MB: 5437.1 | uss_MB: 4385.0 | child_rss_MB: 10498.2 | allocated_MB: 605.9 | reserved_MB: 28584.0 | inactive_split_MB: 314.1 +rank4 [2026-10-27 17:14:55,830] INFO : Step 16000/100000 rank4 | Train Loss: 14.0060 | Train Loss Unweighted: 4.5724 | Train Loss Components: Classification: 0.0150 | Classification_binary: 3.5827 | Regression_cos_phi: 0.0469 | Regression_energy: 0.4276 | Regression_eta: 0.0760 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0517 | Group Train Loss Unweighted (window mean): physical_pu: 3.7654 | physical_nopu: 4.6500 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3929 | Classification_binary: 3.5850 | Regression_cos_phi: 1.5208 | Regression_energy: 1.7948 | Regression_eta: 0.6253 | Regression_pt: 2.1885 | Regression_sin_phi: 2.0231 | LR: 3.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.4624s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0984 | max_forward: 0.1749 | max_loss: 0.0039 | max_backward: 0.0976 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 2.1733 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5435.1 | uss_MB: 4384.3 | child_rss_MB: 10495.1 | allocated_MB: 607.6 | reserved_MB: 28978.0 | inactive_split_MB: 330.4 +rank4 [2026-10-27 17:18:33,642] INFO : Step 17000/100000 rank4 | Train Loss: 14.2557 | Train Loss Unweighted: 4.6663 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6292 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4540 | Regression_eta: 0.0715 | Regression_pt: 0.3998 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.7576 | physical_nopu: 4.6430 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5279 | Classification_binary: 3.6201 | Regression_cos_phi: 1.4985 | Regression_energy: 3.1432 | Regression_eta: 1.1135 | Regression_pt: 3.8253 | Regression_sin_phi: 1.8279 | LR: 3.75e-04 | DataLoad Time: 0.0030s | Model Forward Time: 3.9701s | Phase Times: max_pre_step_barrier: 0.5705 | max_batch_to_device: 0.0043 | max_forward: 0.0887 | max_loss: 0.0037 | max_backward: 0.1154 | max_optimizer_step: 0.2542 | max_tensorboard_logging: 2.1801 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5421.0 | uss_MB: 4384.2 | child_rss_MB: 10406.9 | allocated_MB: 609.7 | reserved_MB: 29500.0 | inactive_split_MB: 302.1 +rank4 [2026-10-27 17:22:18,273] INFO : Step 18000/100000 rank4 | Train Loss: 13.2896 | Train Loss Unweighted: 4.5062 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6504 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3607 | Regression_eta: 0.0732 | Regression_pt: 0.3062 | Regression_sin_phi: 0.0530 | Group Train Loss Unweighted (window mean): physical_pu: 3.7500 | physical_nopu: 4.6286 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5718 | Classification_binary: 3.4969 | Regression_cos_phi: 1.5882 | Regression_energy: 1.8613 | Regression_eta: 1.2363 | Regression_pt: 2.1618 | Regression_sin_phi: 1.9860 | LR: 3.72e-04 | DataLoad Time: 0.0012s | Model Forward Time: 10.0059s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0832 | max_loss: 0.0037 | max_backward: 0.0999 | max_optimizer_step: 0.1725 | max_tensorboard_logging: 8.8254 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 5416.7 | uss_MB: 4384.0 | child_rss_MB: 10363.7 | allocated_MB: 608.8 | reserved_MB: 29500.0 | inactive_split_MB: 196.1 +rank4 [2026-10-27 17:25:51,407] INFO : Step 19000/100000 rank4 | Train Loss: 15.2449 | Train Loss Unweighted: 4.8480 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5669 | Regression_cos_phi: 0.0399 | Regression_energy: 0.5792 | Regression_eta: 0.0602 | Regression_pt: 0.5415 | Regression_sin_phi: 0.0443 | Group Train Loss Unweighted (window mean): physical_pu: 3.7399 | physical_nopu: 4.6217 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4623 | Classification_binary: 3.5401 | Regression_cos_phi: 1.1225 | Regression_energy: 2.0907 | Regression_eta: 0.7472 | Regression_pt: 2.6007 | Regression_sin_phi: 2.0238 | LR: 3.69e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.1644s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0008 | max_forward: 0.0818 | max_loss: 0.0762 | max_backward: 0.1759 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 1.9822 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5410.7 | uss_MB: 4384.1 | child_rss_MB: 10330.8 | allocated_MB: 607.1 | reserved_MB: 29500.0 | inactive_split_MB: 236.2 +rank4 [2026-10-27 17:29:45,026] INFO : Step 20000/100000 rank4 | Train Loss: 13.7275 | Train Loss Unweighted: 4.6495 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.7120 | Regression_cos_phi: 0.0510 | Regression_energy: 0.4061 | Regression_eta: 0.0646 | Regression_pt: 0.3531 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.7387 | physical_nopu: 4.6038 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3522 | Classification_binary: 3.5761 | Regression_cos_phi: 2.0958 | Regression_energy: 3.8858 | Regression_eta: 0.9707 | Regression_pt: 4.8593 | Regression_sin_phi: 2.0351 | LR: 3.66e-04 | DataLoad Time: 0.0029s | Model Forward Time: 5.4930s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0007 | max_forward: 0.0851 | max_loss: 0.0039 | max_backward: 0.2016 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 4.2823 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5411.1 | uss_MB: 4383.8 | child_rss_MB: 10307.1 | allocated_MB: 609.5 | reserved_MB: 29500.0 | inactive_split_MB: 252.8 +rank4 [2026-10-27 17:29:45,969] INFO : Running validation on rank4 +rank4 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 17:31:42,604] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 17:31:43,254] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 17:32:09,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 17:32:10,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 17:32:10,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 17:32:11,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 17:32:11,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 17:32:12,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 17:32:12,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 17:32:13,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 17:32:23,246] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 17:32:23,636] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 17:32:24,023] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 17:32:39,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 17:32:40,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 17:32:40,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 17:32:41,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 17:32:41,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 17:32:42,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 17:32:42,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 17:32:43,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 17:32:53,425] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-27 17:32:53,432] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 17:32:53,822] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 17:33:09,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 17:33:09,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 17:33:10,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 17:33:10,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 17:33:11,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 17:33:11,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 17:33:12,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 17:33:12,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 17:33:22,745] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 17:37:33,418] INFO : Step 21000/100000 rank4 | Train Loss: 13.1647 | Train Loss Unweighted: 4.3967 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.5106 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3742 | Regression_eta: 0.0796 | Regression_pt: 0.3229 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.7328 | physical_nopu: 4.5933 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5404 | Classification_binary: 3.4746 | Regression_cos_phi: 1.9672 | Regression_energy: 1.9767 | Regression_eta: 1.0994 | Regression_pt: 2.3039 | Regression_sin_phi: 1.9070 | LR: 3.62e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.6676s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0483 | max_forward: 0.1678 | max_loss: 0.0037 | max_backward: 0.1430 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 2.3569 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5516.5 | uss_MB: 4216.2 | child_rss_MB: 10241.6 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 238.6 +rank4 [2026-10-27 17:41:10,019] INFO : Step 22000/100000 rank4 | Train Loss: 15.3802 | Train Loss Unweighted: 4.7914 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.5311 | Regression_cos_phi: 0.0475 | Regression_energy: 0.5593 | Regression_eta: 0.0730 | Regression_pt: 0.5219 | Regression_sin_phi: 0.0434 | Group Train Loss Unweighted (window mean): physical_pu: 3.7290 | physical_nopu: 4.5858 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4477 | Classification_binary: 3.4722 | Regression_cos_phi: 1.6308 | Regression_energy: 2.0018 | Regression_eta: 0.7999 | Regression_pt: 2.3450 | Regression_sin_phi: 2.1913 | LR: 3.59e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.1744s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0905 | max_loss: 0.0038 | max_backward: 0.1361 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 2.0169 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5517.5 | uss_MB: 4498.8 | child_rss_MB: 10247.2 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 321.0 +rank4 [2026-10-27 17:44:47,651] INFO : Step 23000/100000 rank4 | Train Loss: 12.9848 | Train Loss Unweighted: 4.5226 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.7377 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3326 | Regression_eta: 0.0739 | Regression_pt: 0.2617 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.7288 | physical_nopu: 4.5774 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3443 | Classification_binary: 3.8474 | Regression_cos_phi: 1.7797 | Regression_energy: 1.7344 | Regression_eta: 0.8389 | Regression_pt: 2.0075 | Regression_sin_phi: 1.8137 | LR: 3.55e-04 | DataLoad Time: 0.0074s | Model Forward Time: 3.1583s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0994 | max_loss: 0.0039 | max_backward: 0.1244 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.9838 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 5517.5 | uss_MB: 4498.0 | child_rss_MB: 10251.2 | allocated_MB: 608.2 | reserved_MB: 29422.0 | inactive_split_MB: 218.5 +rank4 [2026-10-27 17:48:44,022] INFO : Step 24000/100000 rank4 | Train Loss: 16.3896 | Train Loss Unweighted: 4.8949 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4510 | Regression_cos_phi: 0.0465 | Regression_energy: 0.6613 | Regression_eta: 0.0617 | Regression_pt: 0.6132 | Regression_sin_phi: 0.0453 | Group Train Loss Unweighted (window mean): physical_pu: 3.7170 | physical_nopu: 4.5767 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5015 | Classification_binary: 3.2997 | Regression_cos_phi: 1.6743 | Regression_energy: 3.1368 | Regression_eta: 0.8031 | Regression_pt: 3.9592 | Regression_sin_phi: 2.0287 | LR: 3.51e-04 | DataLoad Time: 0.0027s | Model Forward Time: 3.1023s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0398 | max_forward: 0.1314 | max_loss: 0.0037 | max_backward: 0.0994 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 1.8290 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5518.7 | uss_MB: 4498.4 | child_rss_MB: 10238.8 | allocated_MB: 606.3 | reserved_MB: 29422.0 | inactive_split_MB: 288.6 +rank4 [2026-10-27 17:52:29,098] INFO : Step 25000/100000 rank4 | Train Loss: 13.0062 | Train Loss Unweighted: 4.4048 | Train Loss Components: Classification: 0.0114 | Classification_binary: 3.6384 | Regression_cos_phi: 0.0552 | Regression_energy: 0.3153 | Regression_eta: 0.0680 | Regression_pt: 0.2574 | Regression_sin_phi: 0.0590 | Group Train Loss Unweighted (window mean): physical_pu: 3.7138 | physical_nopu: 4.5644 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3864 | Classification_binary: 3.6232 | Regression_cos_phi: 2.1148 | Regression_energy: 2.0418 | Regression_eta: 1.2674 | Regression_pt: 2.0479 | Regression_sin_phi: 2.2729 | LR: 3.47e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.5989s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0007 | max_forward: 0.1394 | max_loss: 0.0036 | max_backward: 0.2004 | max_optimizer_step: 0.1603 | max_tensorboard_logging: 1.3791 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5517.7 | uss_MB: 4498.3 | child_rss_MB: 10255.6 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 213.9 +rank4 [2026-10-27 17:56:10,619] INFO : Step 26000/100000 rank4 | Train Loss: 13.5469 | Train Loss Unweighted: 4.4523 | Train Loss Components: Classification: 0.0185 | Classification_binary: 3.5246 | Regression_cos_phi: 0.0498 | Regression_energy: 0.3935 | Regression_eta: 0.0732 | Regression_pt: 0.3461 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5624 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6457 | Classification_binary: 3.4423 | Regression_cos_phi: 1.3859 | Regression_energy: 1.8466 | Regression_eta: 0.9628 | Regression_pt: 2.2166 | Regression_sin_phi: 1.3719 | LR: 3.43e-04 | DataLoad Time: 0.0010s | Model Forward Time: 3.1222s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0838 | max_forward: 0.1679 | max_loss: 0.0039 | max_backward: 0.0991 | max_optimizer_step: 0.2674 | max_tensorboard_logging: 1.6325 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5515.7 | uss_MB: 4498.4 | child_rss_MB: 10248.9 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 201.9 +rank4 [2026-10-27 17:59:48,627] INFO : Step 27000/100000 rank4 | Train Loss: 17.1780 | Train Loss Unweighted: 5.0531 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.5146 | Regression_cos_phi: 0.0448 | Regression_energy: 0.6955 | Regression_eta: 0.0778 | Regression_pt: 0.6583 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5579 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5999 | Classification_binary: 3.3518 | Regression_cos_phi: 1.7669 | Regression_energy: 4.6667 | Regression_eta: 1.2124 | Regression_pt: 5.9199 | Regression_sin_phi: 1.7757 | LR: 3.39e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.4479s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1019 | max_loss: 0.0489 | max_backward: 0.1596 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5507.7 | uss_MB: 4499.5 | child_rss_MB: 10244.8 | allocated_MB: 611.7 | reserved_MB: 30248.0 | inactive_split_MB: 336.3 +rank4 [2026-10-27 18:03:26,754] INFO : Step 28000/100000 rank4 | Train Loss: 12.6721 | Train Loss Unweighted: 4.3436 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5550 | Regression_cos_phi: 0.0488 | Regression_energy: 0.3264 | Regression_eta: 0.0754 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7057 | physical_nopu: 4.5482 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4015 | Classification_binary: 3.3761 | Regression_cos_phi: 1.6376 | Regression_energy: 1.4882 | Regression_eta: 1.0012 | Regression_pt: 1.8338 | Regression_sin_phi: 1.6789 | LR: 3.35e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.0164s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0898 | max_loss: 0.0057 | max_backward: 0.1164 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.8021 | max_tensorboard_flush: 0.0040 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 5489.8 | uss_MB: 4496.9 | child_rss_MB: 10097.2 | allocated_MB: 606.6 | reserved_MB: 30248.0 | inactive_split_MB: 278.7 +rank4 [2026-10-27 18:07:25,425] INFO : Step 29000/100000 rank4 | Train Loss: 14.8403 | Train Loss Unweighted: 4.5753 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.4458 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4901 | Regression_eta: 0.0767 | Regression_pt: 0.4459 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.7068 | physical_nopu: 4.5439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4332 | Classification_binary: 3.5297 | Regression_cos_phi: 1.6200 | Regression_energy: 2.5403 | Regression_eta: 1.0165 | Regression_pt: 2.9545 | Regression_sin_phi: 2.2454 | LR: 3.30e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.3384s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1464 | max_loss: 0.0037 | max_backward: 0.0935 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 1.1535 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5489.8 | uss_MB: 4498.4 | child_rss_MB: 10094.9 | allocated_MB: 607.4 | reserved_MB: 30248.0 | inactive_split_MB: 313.3 +rank4 [2026-10-27 18:11:12,494] INFO : Step 30000/100000 rank4 | Train Loss: 13.2916 | Train Loss Unweighted: 4.4954 | Train Loss Components: Classification: 0.0126 | Classification_binary: 3.5989 | Regression_cos_phi: 0.0482 | Regression_energy: 0.3865 | Regression_eta: 0.0707 | Regression_pt: 0.3291 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7005 | physical_nopu: 4.5378 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5269 | Classification_binary: 3.6796 | Regression_cos_phi: 1.7306 | Regression_energy: 2.0692 | Regression_eta: 1.0984 | Regression_pt: 2.4970 | Regression_sin_phi: 2.1526 | LR: 3.26e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.7228s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0805 | max_loss: 0.0038 | max_backward: 0.0926 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 1.5279 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 5479.8 | uss_MB: 4498.3 | child_rss_MB: 10016.6 | allocated_MB: 608.2 | reserved_MB: 30760.0 | inactive_split_MB: 232.9 +rank4 [2026-10-27 18:11:13,431] INFO : Running validation on rank4 +rank4 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 18:11:13,432] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 18:11:13,432] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 18:13:07,587] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 18:13:08,047] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 18:13:34,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 18:13:35,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 18:13:35,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 18:13:36,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 18:13:36,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 18:13:37,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 18:13:37,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 18:13:38,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 18:13:48,074] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 18:13:48,509] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 18:13:49,356] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 18:14:05,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 18:14:05,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 18:14:06,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 18:14:06,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 18:14:07,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 18:14:07,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 18:14:08,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 18:14:08,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 18:14:18,666] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 18:14:18,746] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 18:14:19,221] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 18:14:34,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 18:14:34,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 18:14:35,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 18:14:35,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 18:14:36,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 18:14:36,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 18:14:37,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 18:14:37,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 18:14:47,743] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-27 18:19:00,303] INFO : Step 31000/100000 rank4 | Train Loss: 12.7750 | Train Loss Unweighted: 4.3889 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5406 | Regression_cos_phi: 0.0465 | Regression_energy: 0.3622 | Regression_eta: 0.0700 | Regression_pt: 0.3101 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6964 | physical_nopu: 4.5360 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4516 | Classification_binary: 3.6795 | Regression_cos_phi: 1.4033 | Regression_energy: 1.8906 | Regression_eta: 0.9273 | Regression_pt: 2.2952 | Regression_sin_phi: 1.3705 | LR: 3.21e-04 | DataLoad Time: 0.0006s | Model Forward Time: 3.3250s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.2346 | max_loss: 0.0037 | max_backward: 0.1255 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 1.8384 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1395 | Memory: rss_MB: 6773.5 | uss_MB: 5791.8 | child_rss_MB: 10011.8 | allocated_MB: 609.3 | reserved_MB: 29142.0 | inactive_split_MB: 278.5 +rank4 [2026-10-27 18:22:44,787] INFO : Step 32000/100000 rank4 | Train Loss: 14.3617 | Train Loss Unweighted: 4.6225 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4934 | Regression_cos_phi: 0.0448 | Regression_energy: 0.5043 | Regression_eta: 0.0718 | Regression_pt: 0.4507 | Regression_sin_phi: 0.0414 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5314 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5696 | Classification_binary: 3.4504 | Regression_cos_phi: 2.0342 | Regression_energy: 3.6253 | Regression_eta: 0.9253 | Regression_pt: 4.4290 | Regression_sin_phi: 1.7646 | LR: 3.16e-04 | DataLoad Time: 0.0103s | Model Forward Time: 2.5623s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1373 | max_loss: 0.0041 | max_backward: 0.2040 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 1.2065 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5556.1 | uss_MB: 4574.8 | child_rss_MB: 10008.9 | allocated_MB: 606.1 | reserved_MB: 29142.0 | inactive_split_MB: 511.2 +rank4 [2026-10-27 18:26:33,026] INFO : Step 33000/100000 rank4 | Train Loss: 12.0000 | Train Loss Unweighted: 3.7115 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9674 | Regression_cos_phi: 0.0474 | Regression_energy: 0.2863 | Regression_eta: 0.1056 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4891 | Classification_binary: 2.9420 | Regression_cos_phi: 1.7566 | Regression_energy: 1.5065 | Regression_eta: 1.6119 | Regression_pt: 1.6589 | Regression_sin_phi: 1.8403 | LR: 3.12e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.5988s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0030 | max_forward: 0.1619 | max_loss: 0.0119 | max_backward: 0.2845 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.1927 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5557.1 | uss_MB: 4556.3 | child_rss_MB: 10013.2 | allocated_MB: 623.7 | reserved_MB: 29572.0 | inactive_split_MB: 1441.5 +rank4 [2026-10-27 18:30:15,971] INFO : Step 34000/100000 rank4 | Train Loss: 15.0627 | Train Loss Unweighted: 4.6899 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4670 | Regression_cos_phi: 0.0454 | Regression_energy: 0.5438 | Regression_eta: 0.0626 | Regression_pt: 0.5082 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6889 | physical_nopu: 4.5215 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5697 | Classification_binary: 3.3519 | Regression_cos_phi: 1.7607 | Regression_energy: 3.0826 | Regression_eta: 1.0787 | Regression_pt: 3.8278 | Regression_sin_phi: 2.3802 | LR: 3.07e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.3753s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0007 | max_forward: 0.1346 | max_loss: 0.0037 | max_backward: 0.1938 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1506 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5552.6 | uss_MB: 4573.6 | child_rss_MB: 9984.9 | allocated_MB: 609.6 | reserved_MB: 29572.0 | inactive_split_MB: 230.4 +rank4 [2026-10-27 18:33:55,272] INFO : Step 35000/100000 rank4 | Train Loss: 12.2019 | Train Loss Unweighted: 4.3540 | Train Loss Components: Classification: 0.0089 | Classification_binary: 3.6426 | Regression_cos_phi: 0.0532 | Regression_energy: 0.2965 | Regression_eta: 0.0664 | Regression_pt: 0.2370 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6854 | physical_nopu: 4.5184 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3964 | Classification_binary: 3.6495 | Regression_cos_phi: 2.2161 | Regression_energy: 1.8427 | Regression_eta: 0.9396 | Regression_pt: 2.0248 | Regression_sin_phi: 1.9773 | LR: 3.02e-04 | DataLoad Time: 0.0013s | Model Forward Time: 2.6477s | Phase Times: max_pre_step_barrier: 0.0511 | max_batch_to_device: 0.0857 | max_forward: 0.1832 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 1.2656 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 5548.6 | uss_MB: 4574.4 | child_rss_MB: 9918.7 | allocated_MB: 608.2 | reserved_MB: 29572.0 | inactive_split_MB: 245.8 +rank4 [2026-10-27 18:37:33,850] INFO : Step 36000/100000 rank4 | Train Loss: 12.7130 | Train Loss Unweighted: 4.3877 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.5479 | Regression_cos_phi: 0.0433 | Regression_energy: 0.3521 | Regression_eta: 0.0636 | Regression_pt: 0.3167 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.6860 | physical_nopu: 4.5137 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5501 | Classification_binary: 3.4185 | Regression_cos_phi: 1.5424 | Regression_energy: 1.7616 | Regression_eta: 0.8692 | Regression_pt: 2.1072 | Regression_sin_phi: 1.3911 | LR: 2.97e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.5817s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0007 | max_forward: 0.1685 | max_loss: 0.0137 | max_backward: 0.0851 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.2906 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 5532.6 | uss_MB: 4487.8 | child_rss_MB: 9818.0 | allocated_MB: 609.0 | reserved_MB: 29572.0 | inactive_split_MB: 268.4 +rank4 [2026-10-27 18:41:12,499] INFO : Step 37000/100000 rank4 | Train Loss: 16.1273 | Train Loss Unweighted: 4.7522 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4049 | Regression_cos_phi: 0.0495 | Regression_energy: 0.5975 | Regression_eta: 0.0881 | Regression_pt: 0.5494 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6829 | physical_nopu: 4.5107 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5338 | Classification_binary: 3.2041 | Regression_cos_phi: 2.0995 | Regression_energy: 5.3689 | Regression_eta: 2.8910 | Regression_pt: 6.7654 | Regression_sin_phi: 2.8149 | LR: 2.91e-04 | DataLoad Time: 0.0027s | Model Forward Time: 2.3164s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0884 | max_loss: 0.0036 | max_backward: 0.0963 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 1.0635 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0656 | Memory: rss_MB: 5529.2 | uss_MB: 4574.6 | child_rss_MB: 9790.1 | allocated_MB: 605.6 | reserved_MB: 29572.0 | inactive_split_MB: 233.2 +rank4 [2026-10-27 18:44:44,282] INFO : Step 38000/100000 rank4 | Train Loss: 12.5142 | Train Loss Unweighted: 4.4218 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.6290 | Regression_cos_phi: 0.0475 | Regression_energy: 0.3376 | Regression_eta: 0.0660 | Regression_pt: 0.2827 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6875 | physical_nopu: 4.5136 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3289 | Classification_binary: 3.8131 | Regression_cos_phi: 1.6946 | Regression_energy: 1.5898 | Regression_eta: 0.8186 | Regression_pt: 1.7319 | Regression_sin_phi: 2.1465 | LR: 2.86e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.4325s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.0807 | max_loss: 0.0039 | max_backward: 0.1583 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.2386 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5519.2 | uss_MB: 4574.3 | child_rss_MB: 9713.9 | allocated_MB: 607.7 | reserved_MB: 29572.0 | inactive_split_MB: 203.1 +rank4 [2026-10-27 18:48:30,203] INFO : Step 39000/100000 rank4 | Train Loss: 13.9834 | Train Loss Unweighted: 4.4509 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4391 | Regression_cos_phi: 0.0477 | Regression_energy: 0.4363 | Regression_eta: 0.0792 | Regression_pt: 0.3839 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.6802 | physical_nopu: 4.5058 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4760 | Classification_binary: 3.4210 | Regression_cos_phi: 1.6358 | Regression_energy: 2.1321 | Regression_eta: 1.6125 | Regression_pt: 2.6536 | Regression_sin_phi: 1.6752 | LR: 2.81e-04 | DataLoad Time: 0.0027s | Model Forward Time: 3.8720s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.0950 | max_loss: 0.0039 | max_backward: 0.1102 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 2.6970 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5506.6 | uss_MB: 4574.6 | child_rss_MB: 9623.2 | allocated_MB: 611.4 | reserved_MB: 29682.0 | inactive_split_MB: 276.6 +rank4 [2026-10-27 18:52:07,824] INFO : Step 40000/100000 rank4 | Train Loss: 15.4467 | Train Loss Unweighted: 4.8241 | Train Loss Components: Classification: 0.0121 | Classification_binary: 3.5538 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5661 | Regression_eta: 0.0744 | Regression_pt: 0.5256 | Regression_sin_phi: 0.0462 | Group Train Loss Unweighted (window mean): physical_pu: 3.6754 | physical_nopu: 4.5045 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3460 | Classification_binary: 3.4478 | Regression_cos_phi: 1.5546 | Regression_energy: 2.6212 | Regression_eta: 1.3109 | Regression_pt: 3.1467 | Regression_sin_phi: 1.6190 | LR: 2.76e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.4736s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.1251 | max_loss: 0.0037 | max_backward: 0.0913 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2799 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5491.6 | uss_MB: 4574.5 | child_rss_MB: 9510.6 | allocated_MB: 607.6 | reserved_MB: 29682.0 | inactive_split_MB: 251.4 +rank4 [2026-10-27 18:52:08,683] INFO : Running validation on rank4 +rank4 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 18:54:11,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 18:54:12,296] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 18:54:37,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 18:54:37,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 18:54:38,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 18:54:38,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 18:54:39,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 18:54:39,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 18:54:40,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 18:54:40,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 18:54:50,959] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-27 18:54:51,320] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 18:54:51,718] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 18:55:07,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 18:55:07,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 18:55:08,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 18:55:08,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 18:55:09,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 18:55:09,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 18:55:10,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 18:55:10,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 18:55:20,708] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 18:55:21,167] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 18:55:21,564] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 18:55:36,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 18:55:37,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 18:55:37,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 18:55:38,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 18:55:38,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 18:55:39,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 18:55:40,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 18:55:40,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 18:55:50,508] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 19:00:08,492] INFO : Step 41000/100000 rank4 | Train Loss: 13.2101 | Train Loss Unweighted: 4.4466 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5620 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3777 | Regression_eta: 0.0720 | Regression_pt: 0.3238 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6774 | physical_nopu: 4.4925 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4202 | Classification_binary: 3.6447 | Regression_cos_phi: 1.6793 | Regression_energy: 2.0560 | Regression_eta: 0.7436 | Regression_pt: 2.4622 | Regression_sin_phi: 1.4911 | LR: 2.70e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.5582s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0904 | max_forward: 0.2227 | max_loss: 0.0038 | max_backward: 0.1239 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 2.1580 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6748.4 | uss_MB: 5831.9 | child_rss_MB: 9514.0 | allocated_MB: 608.3 | reserved_MB: 27230.0 | inactive_split_MB: 387.2 +rank4 [2026-10-27 19:03:45,853] INFO : Step 42000/100000 rank4 | Train Loss: 13.7061 | Train Loss Unweighted: 4.4760 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4108 | Regression_cos_phi: 0.0459 | Regression_energy: 0.4702 | Regression_eta: 0.0614 | Regression_pt: 0.4341 | Regression_sin_phi: 0.0374 | Group Train Loss Unweighted (window mean): physical_pu: 3.6711 | physical_nopu: 4.4967 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4277 | Classification_binary: 3.4777 | Regression_cos_phi: 1.7852 | Regression_energy: 1.8778 | Regression_eta: 0.8051 | Regression_pt: 2.1973 | Regression_sin_phi: 1.3647 | LR: 2.65e-04 | DataLoad Time: 0.0080s | Model Forward Time: 2.9848s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0008 | max_forward: 0.1920 | max_loss: 0.0038 | max_backward: 0.1427 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 1.6213 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6739.4 | uss_MB: 5628.8 | child_rss_MB: 9436.4 | allocated_MB: 609.4 | reserved_MB: 27632.0 | inactive_split_MB: 509.5 +rank4 [2026-10-27 19:07:23,524] INFO : Step 43000/100000 rank4 | Train Loss: 11.9489 | Train Loss Unweighted: 3.6938 | Train Loss Components: Classification: 0.0162 | Classification_binary: 2.9682 | Regression_cos_phi: 0.0472 | Regression_energy: 0.2764 | Regression_eta: 0.1089 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6702 | physical_nopu: 4.4944 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4508 | Classification_binary: 2.9571 | Regression_cos_phi: 1.7165 | Regression_energy: 1.4145 | Regression_eta: 1.5993 | Regression_pt: 1.5499 | Regression_sin_phi: 1.8558 | LR: 2.59e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.8331s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0032 | max_forward: 0.2195 | max_loss: 0.0114 | max_backward: 0.4513 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 2.0889 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0865 | Memory: rss_MB: 5560.5 | uss_MB: 3524.0 | child_rss_MB: 9348.2 | allocated_MB: 619.9 | reserved_MB: 27918.0 | inactive_split_MB: 1305.5 +rank4 [2026-10-27 19:11:14,852] INFO : Step 44000/100000 rank4 | Train Loss: 15.8790 | Train Loss Unweighted: 4.7893 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4610 | Regression_cos_phi: 0.0438 | Regression_energy: 0.5944 | Regression_eta: 0.0747 | Regression_pt: 0.5488 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6681 | physical_nopu: 4.4856 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4793 | Classification_binary: 3.4007 | Regression_cos_phi: 1.4430 | Regression_energy: 2.2144 | Regression_eta: 1.1469 | Regression_pt: 2.5215 | Regression_sin_phi: 1.5912 | LR: 2.54e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.1649s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0980 | max_loss: 0.0531 | max_backward: 0.1371 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.9403 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5553.5 | uss_MB: 4666.8 | child_rss_MB: 9287.7 | allocated_MB: 606.4 | reserved_MB: 31154.0 | inactive_split_MB: 295.0 +rank4 [2026-10-27 19:15:01,382] INFO : Step 45000/100000 rank4 | Train Loss: 13.8786 | Train Loss Unweighted: 4.5241 | Train Loss Components: Classification: 0.0100 | Classification_binary: 3.5822 | Regression_cos_phi: 0.0536 | Regression_energy: 0.4068 | Regression_eta: 0.0847 | Regression_pt: 0.3344 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 3.6667 | physical_nopu: 4.4836 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2573 | Classification_binary: 3.6728 | Regression_cos_phi: 2.1506 | Regression_energy: 1.5999 | Regression_eta: 1.0873 | Regression_pt: 1.9371 | Regression_sin_phi: 1.7879 | LR: 2.48e-04 | DataLoad Time: 0.0020s | Model Forward Time: 4.6037s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0009 | max_forward: 0.0951 | max_loss: 0.0815 | max_backward: 0.1837 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 3.3466 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 5551.4 | uss_MB: 4666.0 | child_rss_MB: 9284.1 | allocated_MB: 608.8 | reserved_MB: 31154.0 | inactive_split_MB: 289.8 +rank4 [2026-10-27 19:18:42,715] INFO : Step 46000/100000 rank4 | Train Loss: 13.1587 | Train Loss Unweighted: 4.3779 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.4817 | Regression_cos_phi: 0.0459 | Regression_energy: 0.3805 | Regression_eta: 0.0668 | Regression_pt: 0.3375 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.6620 | physical_nopu: 4.4831 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4287 | Classification_binary: 3.4374 | Regression_cos_phi: 1.7015 | Regression_energy: 2.1448 | Regression_eta: 1.1217 | Regression_pt: 2.4529 | Regression_sin_phi: 2.0186 | LR: 2.43e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.3331s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0924 | max_forward: 0.1910 | max_loss: 0.0039 | max_backward: 0.1028 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 2.0151 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5545.4 | uss_MB: 4666.2 | child_rss_MB: 9248.1 | allocated_MB: 609.2 | reserved_MB: 31154.0 | inactive_split_MB: 277.4 +rank4 [2026-10-27 19:22:31,522] INFO : Step 47000/100000 rank4 | Train Loss: 15.2118 | Train Loss Unweighted: 4.6831 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3624 | Regression_cos_phi: 0.0409 | Regression_energy: 0.5987 | Regression_eta: 0.0659 | Regression_pt: 0.5598 | Regression_sin_phi: 0.0405 | Group Train Loss Unweighted (window mean): physical_pu: 3.6609 | physical_nopu: 4.4795 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4782 | Classification_binary: 3.2879 | Regression_cos_phi: 1.3615 | Regression_energy: 3.9487 | Regression_eta: 1.6272 | Regression_pt: 5.0334 | Regression_sin_phi: 1.4139 | LR: 2.37e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.2741s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0777 | max_forward: 0.1864 | max_loss: 0.0077 | max_backward: 0.1105 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 1.8392 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5533.2 | uss_MB: 4665.8 | child_rss_MB: 9129.7 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 305.0 +rank4 [2026-10-27 19:26:21,057] INFO : Step 48000/100000 rank4 | Train Loss: 12.0455 | Train Loss Unweighted: 4.3014 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.6150 | Regression_cos_phi: 0.0526 | Regression_energy: 0.2805 | Regression_eta: 0.0652 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6617 | physical_nopu: 4.4726 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3436 | Classification_binary: 3.7864 | Regression_cos_phi: 2.2728 | Regression_energy: 1.8808 | Regression_eta: 1.4670 | Regression_pt: 2.0139 | Regression_sin_phi: 2.3625 | LR: 2.31e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.3651s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0531 | max_forward: 0.1607 | max_loss: 0.0044 | max_backward: 0.0997 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 2.0345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5534.2 | uss_MB: 4664.3 | child_rss_MB: 9129.2 | allocated_MB: 606.8 | reserved_MB: 31154.0 | inactive_split_MB: 262.3 +rank4 [2026-10-27 19:30:06,213] INFO : Step 49000/100000 rank4 | Train Loss: 13.3973 | Train Loss Unweighted: 4.4035 | Train Loss Components: Classification: 0.0168 | Classification_binary: 3.5033 | Regression_cos_phi: 0.0506 | Regression_energy: 0.3797 | Regression_eta: 0.0752 | Regression_pt: 0.3299 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6566 | physical_nopu: 4.4735 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5144 | Classification_binary: 3.5612 | Regression_cos_phi: 1.8431 | Regression_energy: 2.1503 | Regression_eta: 0.9941 | Regression_pt: 2.5742 | Regression_sin_phi: 1.9231 | LR: 2.26e-04 | DataLoad Time: 0.0010s | Model Forward Time: 4.5496s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0220 | max_forward: 0.1075 | max_loss: 0.0037 | max_backward: 0.1020 | max_optimizer_step: 0.2722 | max_tensorboard_logging: 3.2606 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5532.9 | uss_MB: 4389.0 | child_rss_MB: 9133.5 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 312.3 +rank4 [2026-10-27 19:33:47,826] INFO : Step 50000/100000 rank4 | Train Loss: 14.2027 | Train Loss Unweighted: 4.5299 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4272 | Regression_cos_phi: 0.0432 | Regression_energy: 0.4877 | Regression_eta: 0.0670 | Regression_pt: 0.4429 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4710 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4899 | Classification_binary: 3.4281 | Regression_cos_phi: 1.5048 | Regression_energy: 2.4458 | Regression_eta: 0.8619 | Regression_pt: 2.9836 | Regression_sin_phi: 1.9475 | LR: 2.20e-04 | DataLoad Time: 0.0010s | Model Forward Time: 3.0513s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0964 | max_loss: 0.0590 | max_backward: 0.1520 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.8031 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5519.9 | uss_MB: 4665.8 | child_rss_MB: 9015.1 | allocated_MB: 607.2 | reserved_MB: 31154.0 | inactive_split_MB: 255.6 +rank4 [2026-10-27 19:33:48,728] INFO : Running validation on rank4 +rank4 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 19:35:46,023] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 19:35:46,495] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 19:36:12,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 19:36:12,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 19:36:13,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 19:36:13,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 19:36:14,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 19:36:14,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 19:36:15,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 19:36:15,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 19:36:25,631] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-27 19:36:25,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 19:36:26,397] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 19:36:41,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 19:36:42,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 19:36:42,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 19:36:43,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 19:36:43,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 19:36:44,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 19:36:44,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 19:36:45,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 19:36:55,452] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 19:36:55,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 19:36:55,958] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 19:37:10,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 19:37:11,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 19:37:11,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 19:37:12,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 19:37:12,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 19:37:13,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 19:37:13,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 19:37:14,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 19:37:24,293] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 19:41:35,792] INFO : Step 51000/100000 rank4 | Train Loss: 13.0997 | Train Loss Unweighted: 4.3733 | Train Loss Components: Classification: 0.0129 | Classification_binary: 3.5282 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3598 | Regression_eta: 0.0785 | Regression_pt: 0.2920 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6545 | physical_nopu: 4.4631 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4698 | Classification_binary: 3.5412 | Regression_cos_phi: 1.8887 | Regression_energy: 1.9703 | Regression_eta: 0.8497 | Regression_pt: 2.4719 | Regression_sin_phi: 1.5454 | LR: 2.14e-04 | DataLoad Time: 0.0116s | Model Forward Time: 3.1038s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1313 | max_loss: 0.0271 | max_backward: 0.1689 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.7590 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6858.3 | uss_MB: 6017.9 | child_rss_MB: 8958.7 | allocated_MB: 607.6 | reserved_MB: 29536.0 | inactive_split_MB: 388.7 +rank4 [2026-10-27 19:45:15,993] INFO : Step 52000/100000 rank4 | Train Loss: 11.7496 | Train Loss Unweighted: 3.6574 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9475 | Regression_cos_phi: 0.0469 | Regression_energy: 0.2717 | Regression_eta: 0.1056 | Regression_pt: 0.2231 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6567 | physical_nopu: 4.4685 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4680 | Classification_binary: 3.0033 | Regression_cos_phi: 1.6957 | Regression_energy: 1.4091 | Regression_eta: 1.6521 | Regression_pt: 1.5493 | Regression_sin_phi: 1.9337 | LR: 2.09e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.6140s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.1059 | max_forward: 0.2956 | max_loss: 0.0038 | max_backward: 0.4187 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 1.8177 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6858.3 | uss_MB: 6026.1 | child_rss_MB: 8942.2 | allocated_MB: 619.9 | reserved_MB: 29990.0 | inactive_split_MB: 1078.6 +rank4 [2026-10-27 19:48:51,528] INFO : Step 53000/100000 rank4 | Train Loss: 12.5294 | Train Loss Unweighted: 4.3083 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5302 | Regression_cos_phi: 0.0494 | Regression_energy: 0.3242 | Regression_eta: 0.0722 | Regression_pt: 0.2705 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6530 | physical_nopu: 4.4629 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3509 | Classification_binary: 3.3359 | Regression_cos_phi: 1.6512 | Regression_energy: 1.5272 | Regression_eta: 1.1474 | Regression_pt: 1.7508 | Regression_sin_phi: 1.5539 | LR: 2.03e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.5890s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.1740 | max_loss: 0.0036 | max_backward: 0.1135 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 1.3124 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 5606.5 | uss_MB: 4766.5 | child_rss_MB: 8943.1 | allocated_MB: 608.5 | reserved_MB: 29990.0 | inactive_split_MB: 246.4 +rank4 [2026-10-27 19:52:28,508] INFO : Step 54000/100000 rank4 | Train Loss: 13.6142 | Train Loss Unweighted: 4.3355 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.3742 | Regression_cos_phi: 0.0499 | Regression_energy: 0.4083 | Regression_eta: 0.0736 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4652 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4967 | Classification_binary: 3.3253 | Regression_cos_phi: 1.9990 | Regression_energy: 2.0992 | Regression_eta: 1.4615 | Regression_pt: 2.3415 | Regression_sin_phi: 2.0610 | LR: 1.97e-04 | DataLoad Time: 0.0014s | Model Forward Time: 2.4452s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1029 | max_loss: 0.0156 | max_backward: 0.1263 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2459 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 5600.5 | uss_MB: 4766.6 | child_rss_MB: 8886.2 | allocated_MB: 608.0 | reserved_MB: 29990.0 | inactive_split_MB: 249.1 +rank4 [2026-10-27 19:56:02,481] INFO : Step 55000/100000 rank4 | Train Loss: 14.4463 | Train Loss Unweighted: 4.6356 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5250 | Regression_cos_phi: 0.0497 | Regression_energy: 0.4959 | Regression_eta: 0.0660 | Regression_pt: 0.4403 | Regression_sin_phi: 0.0455 | Group Train Loss Unweighted (window mean): physical_pu: 3.6517 | physical_nopu: 4.4554 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3032 | Classification_binary: 3.7054 | Regression_cos_phi: 1.9951 | Regression_energy: 1.7865 | Regression_eta: 1.0302 | Regression_pt: 1.8396 | Regression_sin_phi: 2.0605 | LR: 1.92e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.9801s | Phase Times: max_pre_step_barrier: 0.0108 | max_batch_to_device: 0.0008 | max_forward: 0.0923 | max_loss: 0.0219 | max_backward: 0.1306 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 1.7796 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5599.8 | uss_MB: 4765.5 | child_rss_MB: 8883.4 | allocated_MB: 610.0 | reserved_MB: 29990.0 | inactive_split_MB: 274.0 +rank4 [2026-10-27 19:59:40,797] INFO : Step 56000/100000 rank4 | Train Loss: 12.4091 | Train Loss Unweighted: 4.2572 | Train Loss Components: Classification: 0.0135 | Classification_binary: 3.4523 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3421 | Regression_eta: 0.0664 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6493 | physical_nopu: 4.4547 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3862 | Classification_binary: 3.5586 | Regression_cos_phi: 1.4746 | Regression_energy: 1.4409 | Regression_eta: 0.8570 | Regression_pt: 1.6394 | Regression_sin_phi: 1.5541 | LR: 1.86e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.0682s | Phase Times: max_pre_step_barrier: 0.3290 | max_batch_to_device: 0.0902 | max_forward: 0.1771 | max_loss: 0.0038 | max_backward: 0.1058 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 1.4273 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5599.8 | uss_MB: 4766.9 | child_rss_MB: 8880.9 | allocated_MB: 608.1 | reserved_MB: 29990.0 | inactive_split_MB: 253.9 +rank4 [2026-10-27 20:03:29,496] INFO : Step 57000/100000 rank4 | Train Loss: 14.9087 | Train Loss Unweighted: 4.5957 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4371 | Regression_cos_phi: 0.0471 | Regression_energy: 0.5028 | Regression_eta: 0.0770 | Regression_pt: 0.4658 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6475 | physical_nopu: 4.4509 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5027 | Classification_binary: 3.4350 | Regression_cos_phi: 1.6227 | Regression_energy: 2.9938 | Regression_eta: 0.9050 | Regression_pt: 3.9262 | Regression_sin_phi: 2.1972 | LR: 1.81e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.4962s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1036 | max_loss: 0.0876 | max_backward: 0.2130 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 1.1951 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5597.8 | uss_MB: 4497.3 | child_rss_MB: 8866.5 | allocated_MB: 608.1 | reserved_MB: 30358.0 | inactive_split_MB: 283.9 +rank4 [2026-10-27 20:07:08,735] INFO : Step 58000/100000 rank4 | Train Loss: 12.4468 | Train Loss Unweighted: 4.2826 | Train Loss Components: Classification: 0.0105 | Classification_binary: 3.5299 | Regression_cos_phi: 0.0545 | Regression_energy: 0.3106 | Regression_eta: 0.0620 | Regression_pt: 0.2643 | Regression_sin_phi: 0.0509 | Group Train Loss Unweighted (window mean): physical_pu: 3.6440 | physical_nopu: 4.4492 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3537 | Classification_binary: 3.6686 | Regression_cos_phi: 1.8274 | Regression_energy: 1.4362 | Regression_eta: 0.9862 | Regression_pt: 1.5702 | Regression_sin_phi: 2.3764 | LR: 1.75e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.5425s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1868 | max_loss: 0.0037 | max_backward: 0.1111 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2860 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5587.6 | uss_MB: 3611.2 | child_rss_MB: 8803.1 | allocated_MB: 608.2 | reserved_MB: 30358.0 | inactive_split_MB: 337.0 +rank4 [2026-10-27 20:10:52,442] INFO : Step 59000/100000 rank4 | Train Loss: 13.1870 | Train Loss Unweighted: 4.3659 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4760 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3771 | Regression_eta: 0.0773 | Regression_pt: 0.3247 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4440 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4529 | Classification_binary: 3.2989 | Regression_cos_phi: 1.4821 | Regression_energy: 1.6481 | Regression_eta: 0.9114 | Regression_pt: 1.9922 | Regression_sin_phi: 1.8406 | LR: 1.70e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.4659s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1005 | max_loss: 0.0036 | max_backward: 0.1062 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 1.2390 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 5582.6 | uss_MB: 4766.7 | child_rss_MB: 8755.9 | allocated_MB: 610.1 | reserved_MB: 31234.0 | inactive_split_MB: 273.9 +rank4 [2026-10-27 20:14:34,818] INFO : Step 60000/100000 rank4 | Train Loss: 15.4000 | Train Loss Unweighted: 4.6905 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.3905 | Regression_cos_phi: 0.0456 | Regression_energy: 0.5805 | Regression_eta: 0.0703 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6406 | physical_nopu: 4.4523 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4618 | Classification_binary: 3.2719 | Regression_cos_phi: 1.4634 | Regression_energy: 2.3163 | Regression_eta: 0.8973 | Regression_pt: 2.8687 | Regression_sin_phi: 1.4997 | LR: 1.64e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.5442s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0757 | max_forward: 0.1624 | max_loss: 0.0040 | max_backward: 0.1315 | max_optimizer_step: 0.2155 | max_tensorboard_logging: 1.2531 | max_tensorboard_flush: 0.0020 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5582.8 | uss_MB: 4766.6 | child_rss_MB: 8760.5 | allocated_MB: 611.6 | reserved_MB: 31234.0 | inactive_split_MB: 318.4 +rank4 [2026-10-27 20:14:35,735] INFO : Running validation on rank4 +rank4 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 20:16:30,818] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 20:16:31,258] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 20:16:55,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 20:16:56,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 20:16:56,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 20:16:57,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 20:16:57,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 20:16:58,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 20:16:58,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 20:16:59,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 20:17:09,146] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 20:17:09,354] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 20:17:09,707] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 20:17:25,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 20:17:25,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 20:17:26,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 20:17:26,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 20:17:27,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 20:17:27,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 20:17:28,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 20:17:28,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 20:17:38,704] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 20:17:39,021] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 20:17:39,451] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 20:17:54,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 20:17:54,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 20:17:55,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 20:17:56,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 20:17:56,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 20:17:57,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 20:17:57,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 20:17:58,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 20:18:08,036] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 20:22:28,768] INFO : Step 61000/100000 rank4 | Train Loss: 12.1474 | Train Loss Unweighted: 4.3289 | Train Loss Components: Classification: 0.0110 | Classification_binary: 3.6236 | Regression_cos_phi: 0.0523 | Regression_energy: 0.2909 | Regression_eta: 0.0654 | Regression_pt: 0.2368 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.6358 | physical_nopu: 4.4439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2739 | Classification_binary: 3.6030 | Regression_cos_phi: 1.6897 | Regression_energy: 1.7118 | Regression_eta: 1.0757 | Regression_pt: 1.9919 | Regression_sin_phi: 1.9858 | LR: 1.59e-04 | DataLoad Time: 0.0078s | Model Forward Time: 3.4728s | Phase Times: max_pre_step_barrier: 0.0115 | max_batch_to_device: 0.0009 | max_forward: 0.2092 | max_loss: 0.0037 | max_backward: 0.1501 | max_optimizer_step: 0.2965 | max_tensorboard_logging: 1.8725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6817.2 | uss_MB: 6009.9 | child_rss_MB: 8755.0 | allocated_MB: 607.8 | reserved_MB: 28822.0 | inactive_split_MB: 222.9 +rank4 [2026-10-27 20:26:14,158] INFO : Step 62000/100000 rank4 | Train Loss: 11.7977 | Train Loss Unweighted: 3.6294 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9084 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2778 | Regression_eta: 0.1039 | Regression_pt: 0.2286 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6417 | physical_nopu: 4.4459 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4434 | Classification_binary: 2.9018 | Regression_cos_phi: 1.7044 | Regression_energy: 1.4869 | Regression_eta: 1.6010 | Regression_pt: 1.6534 | Regression_sin_phi: 1.8711 | LR: 1.54e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.7427s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0033 | max_forward: 0.2219 | max_loss: 0.0037 | max_backward: 0.2965 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1292 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1278 | Memory: rss_MB: 6812.4 | uss_MB: 6017.7 | child_rss_MB: 8760.3 | allocated_MB: 618.8 | reserved_MB: 28822.0 | inactive_split_MB: 888.0 +rank4 [2026-10-27 20:30:05,877] INFO : Step 63000/100000 rank4 | Train Loss: 12.5071 | Train Loss Unweighted: 4.3613 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5997 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3200 | Regression_eta: 0.0719 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6399 | physical_nopu: 4.4447 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4171 | Classification_binary: 3.6233 | Regression_cos_phi: 2.1319 | Regression_energy: 2.3818 | Regression_eta: 1.5633 | Regression_pt: 2.4727 | Regression_sin_phi: 2.0419 | LR: 1.49e-04 | DataLoad Time: 0.0026s | Model Forward Time: 11.0655s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1169 | max_loss: 0.0157 | max_backward: 0.2882 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 9.0944 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6806.2 | uss_MB: 4694.0 | child_rss_MB: 8747.4 | allocated_MB: 610.4 | reserved_MB: 28822.0 | inactive_split_MB: 261.6 +rank4 [2026-10-27 20:33:53,443] INFO : Step 64000/100000 rank4 | Train Loss: 14.0388 | Train Loss Unweighted: 4.5035 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.4200 | Regression_cos_phi: 0.0430 | Regression_energy: 0.4752 | Regression_eta: 0.0663 | Regression_pt: 0.4376 | Regression_sin_phi: 0.0442 | Group Train Loss Unweighted (window mean): physical_pu: 3.6357 | physical_nopu: 4.4395 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5650 | Classification_binary: 3.3955 | Regression_cos_phi: 1.4603 | Regression_energy: 3.0967 | Regression_eta: 1.8055 | Regression_pt: 3.7234 | Regression_sin_phi: 1.7526 | LR: 1.43e-04 | DataLoad Time: 0.1728s | Model Forward Time: 6.3162s | Phase Times: max_pre_step_barrier: 0.1731 | max_batch_to_device: 0.0118 | max_forward: 0.1219 | max_loss: 0.0038 | max_backward: 0.1121 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 5.0441 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5600.0 | uss_MB: 4820.3 | child_rss_MB: 8744.3 | allocated_MB: 611.1 | reserved_MB: 29052.0 | inactive_split_MB: 372.9 +rank4 [2026-10-27 20:37:48,516] INFO : Step 65000/100000 rank4 | Train Loss: 16.3374 | Train Loss Unweighted: 4.8919 | Train Loss Components: Classification: 0.0142 | Classification_binary: 3.4871 | Regression_cos_phi: 0.0500 | Regression_energy: 0.6360 | Regression_eta: 0.0628 | Regression_pt: 0.5943 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4335 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3950 | Classification_binary: 3.3472 | Regression_cos_phi: 1.7581 | Regression_energy: 1.8200 | Regression_eta: 0.9812 | Regression_pt: 2.1599 | Regression_sin_phi: 1.5141 | LR: 1.38e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.4886s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.2167 | max_loss: 0.0058 | max_backward: 0.2407 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 1.9060 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5594.0 | uss_MB: 4820.8 | child_rss_MB: 8750.8 | allocated_MB: 609.7 | reserved_MB: 29052.0 | inactive_split_MB: 236.9 +rank4 [2026-10-27 20:41:37,390] INFO : Step 66000/100000 rank4 | Train Loss: 12.8848 | Train Loss Unweighted: 4.2935 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4715 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3451 | Regression_eta: 0.0693 | Regression_pt: 0.2907 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.6349 | physical_nopu: 4.4348 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4653 | Classification_binary: 3.5057 | Regression_cos_phi: 1.5794 | Regression_energy: 1.6496 | Regression_eta: 0.7519 | Regression_pt: 1.9216 | Regression_sin_phi: 1.9095 | LR: 1.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.1922s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0854 | max_loss: 0.1083 | max_backward: 0.2178 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.9218 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5582.1 | uss_MB: 4820.7 | child_rss_MB: 8750.8 | allocated_MB: 611.4 | reserved_MB: 29052.0 | inactive_split_MB: 254.2 +rank4 [2026-10-27 20:45:24,537] INFO : Step 67000/100000 rank4 | Train Loss: 14.2125 | Train Loss Unweighted: 4.5026 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.4493 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4566 | Regression_eta: 0.0671 | Regression_pt: 0.4150 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6352 | physical_nopu: 4.4350 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4946 | Classification_binary: 3.4266 | Regression_cos_phi: 1.3722 | Regression_energy: 2.3990 | Regression_eta: 0.7109 | Regression_pt: 2.9312 | Regression_sin_phi: 1.7751 | LR: 1.28e-04 | DataLoad Time: 0.0028s | Model Forward Time: 3.0117s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0901 | max_loss: 0.0487 | max_backward: 0.1449 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 1.8090 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5575.1 | uss_MB: 4820.8 | child_rss_MB: 8750.2 | allocated_MB: 607.1 | reserved_MB: 29260.0 | inactive_split_MB: 246.6 +rank4 [2026-10-27 20:49:05,126] INFO : Step 68000/100000 rank4 | Train Loss: 12.0487 | Train Loss Unweighted: 4.2633 | Train Loss Components: Classification: 0.0107 | Classification_binary: 3.5135 | Regression_cos_phi: 0.0466 | Regression_energy: 0.3165 | Regression_eta: 0.0645 | Regression_pt: 0.2647 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4387 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2483 | Classification_binary: 3.4132 | Regression_cos_phi: 1.8331 | Regression_energy: 1.8574 | Regression_eta: 1.0040 | Regression_pt: 2.0035 | Regression_sin_phi: 1.7892 | LR: 1.24e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.8889s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0010 | max_forward: 0.0879 | max_loss: 0.0120 | max_backward: 0.1069 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.7176 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5570.9 | uss_MB: 4820.5 | child_rss_MB: 8753.3 | allocated_MB: 609.5 | reserved_MB: 29260.0 | inactive_split_MB: 225.9 +rank4 [2026-10-27 20:52:47,283] INFO : Step 69000/100000 rank4 | Train Loss: 13.1377 | Train Loss Unweighted: 4.3133 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4496 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3632 | Regression_eta: 0.0778 | Regression_pt: 0.3079 | Regression_sin_phi: 0.0541 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4321 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4661 | Classification_binary: 3.4304 | Regression_cos_phi: 1.3603 | Regression_energy: 1.8495 | Regression_eta: 0.6828 | Regression_pt: 2.1625 | Regression_sin_phi: 2.2545 | LR: 1.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 4.9124s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.1207 | max_loss: 0.0509 | max_backward: 0.2012 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 3.5532 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 5571.9 | uss_MB: 4820.8 | child_rss_MB: 8755.8 | allocated_MB: 608.0 | reserved_MB: 29260.0 | inactive_split_MB: 256.4 +rank4 [2026-10-27 20:56:27,513] INFO : Step 70000/100000 rank4 | Train Loss: 15.8477 | Train Loss Unweighted: 4.7317 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3873 | Regression_cos_phi: 0.0498 | Regression_energy: 0.6015 | Regression_eta: 0.0629 | Regression_pt: 0.5683 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6315 | physical_nopu: 4.4280 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5187 | Classification_binary: 3.3022 | Regression_cos_phi: 1.7470 | Regression_energy: 3.8426 | Regression_eta: 0.9210 | Regression_pt: 5.0646 | Regression_sin_phi: 1.8086 | LR: 1.14e-04 | DataLoad Time: 0.0063s | Model Forward Time: 2.9291s | Phase Times: max_pre_step_barrier: 0.0198 | max_batch_to_device: 0.0007 | max_forward: 0.1178 | max_loss: 0.0036 | max_backward: 0.1081 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.7610 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5571.5 | uss_MB: 4819.7 | child_rss_MB: 8751.0 | allocated_MB: 610.3 | reserved_MB: 30150.0 | inactive_split_MB: 260.3 +rank4 [2026-10-27 20:56:28,457] INFO : Running validation on rank4 +rank4 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 20:58:25,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 20:58:26,362] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 20:58:52,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 20:58:52,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 20:58:53,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 20:58:53,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 20:58:54,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 20:58:54,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 20:58:55,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 20:58:55,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 20:59:05,717] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 20:59:05,946] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 20:59:06,379] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 20:59:21,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 20:59:22,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 20:59:22,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 20:59:23,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 20:59:23,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 20:59:24,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 20:59:24,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 20:59:25,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 20:59:35,238] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 20:59:35,862] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 20:59:36,279] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 20:59:51,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 20:59:51,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 20:59:52,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 20:59:52,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 20:59:53,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 20:59:53,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 20:59:54,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 20:59:54,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 21:00:04,966] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 21:04:25,115] INFO : Step 71000/100000 rank4 | Train Loss: 12.0110 | Train Loss Unweighted: 3.6539 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9046 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2881 | Regression_eta: 0.1063 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6306 | physical_nopu: 4.4256 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5102 | Classification_binary: 2.9529 | Regression_cos_phi: 1.7619 | Regression_energy: 1.5328 | Regression_eta: 1.6405 | Regression_pt: 1.7475 | Regression_sin_phi: 1.8893 | LR: 1.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.4582s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0035 | max_forward: 0.1952 | max_loss: 0.0038 | max_backward: 0.3195 | max_optimizer_step: 0.3092 | max_tensorboard_logging: 1.7411 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6798.7 | uss_MB: 6058.3 | child_rss_MB: 8722.6 | allocated_MB: 623.5 | reserved_MB: 27422.0 | inactive_split_MB: 864.2 +rank4 [2026-10-27 21:08:01,993] INFO : Step 72000/100000 rank4 | Train Loss: 15.2878 | Train Loss Unweighted: 4.6447 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.3643 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5735 | Regression_eta: 0.0647 | Regression_pt: 0.5353 | Regression_sin_phi: 0.0449 | Group Train Loss Unweighted (window mean): physical_pu: 3.6275 | physical_nopu: 4.4239 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5195 | Classification_binary: 3.3468 | Regression_cos_phi: 1.6449 | Regression_energy: 3.2091 | Regression_eta: 1.0713 | Regression_pt: 4.0660 | Regression_sin_phi: 1.7003 | LR: 1.05e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.5456s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0232 | max_forward: 0.1174 | max_loss: 0.0036 | max_backward: 0.0921 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 1.2606 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5641.1 | uss_MB: 4896.7 | child_rss_MB: 8682.2 | allocated_MB: 607.6 | reserved_MB: 28586.0 | inactive_split_MB: 303.9 +rank4 [2026-10-27 21:11:50,557] INFO : Step 73000/100000 rank4 | Train Loss: 12.7380 | Train Loss Unweighted: 4.2392 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.4842 | Regression_cos_phi: 0.0575 | Regression_energy: 0.3093 | Regression_eta: 0.0643 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.6309 | physical_nopu: 4.4270 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3445 | Classification_binary: 3.5569 | Regression_cos_phi: 2.1495 | Regression_energy: 1.7787 | Regression_eta: 0.8073 | Regression_pt: 1.9684 | Regression_sin_phi: 2.3333 | LR: 1.01e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.4475s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0985 | max_loss: 0.0632 | max_backward: 0.1654 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2139 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5637.1 | uss_MB: 4897.0 | child_rss_MB: 8681.3 | allocated_MB: 607.0 | reserved_MB: 28586.0 | inactive_split_MB: 227.5 +rank4 [2026-10-27 21:15:34,267] INFO : Step 74000/100000 rank4 | Train Loss: 12.5735 | Train Loss Unweighted: 4.2478 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4076 | Regression_cos_phi: 0.0429 | Regression_energy: 0.3533 | Regression_eta: 0.0699 | Regression_pt: 0.3116 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6307 | physical_nopu: 4.4218 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5134 | Classification_binary: 3.5192 | Regression_cos_phi: 1.4603 | Regression_energy: 1.7753 | Regression_eta: 1.0031 | Regression_pt: 2.1191 | Regression_sin_phi: 1.7150 | LR: 9.68e-05 | DataLoad Time: 0.0022s | Model Forward Time: 2.5463s | Phase Times: max_pre_step_barrier: 0.1687 | max_batch_to_device: 0.0008 | max_forward: 0.0990 | max_loss: 0.0516 | max_backward: 0.1766 | max_optimizer_step: 0.1812 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5626.2 | uss_MB: 4897.3 | child_rss_MB: 8634.5 | allocated_MB: 608.6 | reserved_MB: 28586.0 | inactive_split_MB: 285.4 +rank4 [2026-10-27 21:19:21,651] INFO : Step 75000/100000 rank4 | Train Loss: 15.1687 | Train Loss Unweighted: 4.6098 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.3751 | Regression_cos_phi: 0.0492 | Regression_energy: 0.5530 | Regression_eta: 0.0638 | Regression_pt: 0.5060 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4206 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5545 | Classification_binary: 3.2620 | Regression_cos_phi: 1.6327 | Regression_energy: 3.3923 | Regression_eta: 0.9709 | Regression_pt: 4.1112 | Regression_sin_phi: 1.4939 | LR: 9.27e-05 | DataLoad Time: 0.0016s | Model Forward Time: 2.4839s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1026 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.2820 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5619.2 | uss_MB: 4896.7 | child_rss_MB: 8602.7 | allocated_MB: 610.1 | reserved_MB: 28586.0 | inactive_split_MB: 259.7 +rank4 [2026-10-27 21:23:05,981] INFO : Step 76000/100000 rank4 | Train Loss: 12.2943 | Train Loss Unweighted: 4.2773 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5085 | Regression_cos_phi: 0.0476 | Regression_energy: 0.3262 | Regression_eta: 0.0732 | Regression_pt: 0.2631 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6289 | physical_nopu: 4.4209 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3211 | Classification_binary: 3.5567 | Regression_cos_phi: 1.8165 | Regression_energy: 1.2735 | Regression_eta: 0.8406 | Regression_pt: 1.5314 | Regression_sin_phi: 1.6134 | LR: 8.88e-05 | DataLoad Time: 0.0021s | Model Forward Time: 2.6228s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.1085 | max_optimizer_step: 0.1831 | max_tensorboard_logging: 1.4262 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 5617.3 | uss_MB: 4895.6 | child_rss_MB: 8601.6 | allocated_MB: 608.9 | reserved_MB: 28942.0 | inactive_split_MB: 227.1 +rank4 [2026-10-27 21:27:01,963] INFO : Step 77000/100000 rank4 | Train Loss: 13.8477 | Train Loss Unweighted: 4.4496 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4225 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4510 | Regression_eta: 0.0623 | Regression_pt: 0.4032 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6274 | physical_nopu: 4.4166 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4992 | Classification_binary: 3.7211 | Regression_cos_phi: 2.0342 | Regression_energy: 1.8299 | Regression_eta: 1.0100 | Regression_pt: 2.2539 | Regression_sin_phi: 1.9016 | LR: 8.50e-05 | DataLoad Time: 0.0016s | Model Forward Time: 6.1308s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0839 | max_forward: 0.1754 | max_loss: 0.0036 | max_backward: 0.1103 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 4.6772 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1374 | Memory: rss_MB: 5607.3 | uss_MB: 4838.5 | child_rss_MB: 8502.5 | allocated_MB: 611.0 | reserved_MB: 28942.0 | inactive_split_MB: 369.0 +rank4 [2026-10-27 21:30:52,663] INFO : Step 78000/100000 rank4 | Train Loss: 14.1981 | Train Loss Unweighted: 4.6212 | Train Loss Components: Classification: 0.0133 | Classification_binary: 3.5747 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4573 | Regression_eta: 0.0669 | Regression_pt: 0.4114 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.6241 | physical_nopu: 4.4230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3946 | Classification_binary: 3.5367 | Regression_cos_phi: 1.6614 | Regression_energy: 1.8636 | Regression_eta: 1.1100 | Regression_pt: 2.2000 | Regression_sin_phi: 1.8925 | LR: 8.13e-05 | DataLoad Time: 0.0021s | Model Forward Time: 3.0986s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1149 | max_loss: 0.0636 | max_backward: 0.1914 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5607.3 | uss_MB: 4897.1 | child_rss_MB: 8495.1 | allocated_MB: 606.6 | reserved_MB: 28942.0 | inactive_split_MB: 309.2 +rank4 [2026-10-27 21:34:49,720] INFO : Step 79000/100000 rank4 | Train Loss: 12.3648 | Train Loss Unweighted: 4.2396 | Train Loss Components: Classification: 0.0145 | Classification_binary: 3.4636 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3226 | Regression_eta: 0.0628 | Regression_pt: 0.2792 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6254 | physical_nopu: 4.4183 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4450 | Classification_binary: 3.4299 | Regression_cos_phi: 2.0930 | Regression_energy: 2.2800 | Regression_eta: 1.0089 | Regression_pt: 2.5993 | Regression_sin_phi: 2.9261 | LR: 7.78e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.3389s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0936 | max_forward: 0.1936 | max_loss: 0.0037 | max_backward: 0.1134 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 2.0079 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 5602.4 | uss_MB: 4897.2 | child_rss_MB: 8483.1 | allocated_MB: 609.2 | reserved_MB: 29222.0 | inactive_split_MB: 324.6 +rank4 [2026-10-27 21:38:37,320] INFO : Step 80000/100000 rank4 | Train Loss: 16.1833 | Train Loss Unweighted: 4.8046 | Train Loss Components: Classification: 0.0155 | Classification_binary: 3.4149 | Regression_cos_phi: 0.0453 | Regression_energy: 0.6189 | Regression_eta: 0.0801 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6233 | physical_nopu: 4.4142 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5283 | Classification_binary: 3.4897 | Regression_cos_phi: 1.9169 | Regression_energy: 2.5121 | Regression_eta: 0.9624 | Regression_pt: 3.1356 | Regression_sin_phi: 2.0488 | LR: 7.44e-05 | DataLoad Time: 0.0102s | Model Forward Time: 3.9237s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0008 | max_forward: 0.1071 | max_loss: 0.0037 | max_backward: 0.1380 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 2.6784 | max_tensorboard_flush: 0.0026 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 5596.4 | uss_MB: 4871.2 | child_rss_MB: 8436.2 | allocated_MB: 606.3 | reserved_MB: 29222.0 | inactive_split_MB: 471.7 +rank4 [2026-10-27 21:38:38,322] INFO : Running validation on rank4 +rank4 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 21:38:38,323] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 21:38:38,323] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 21:40:39,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 21:40:39,982] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 21:41:05,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 21:41:06,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 21:41:06,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 21:41:07,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 21:41:07,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 21:41:08,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 21:41:08,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 21:41:09,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 21:41:19,384] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 21:41:19,914] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 21:41:20,297] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 21:41:35,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 21:41:35,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 21:41:36,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 21:41:36,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 21:41:37,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 21:41:37,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 21:41:38,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 21:41:38,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 21:41:48,831] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-27 21:41:49,545] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 21:41:50,007] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 21:42:05,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 21:42:05,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 21:42:06,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 21:42:06,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 21:42:07,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 21:42:07,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 21:42:08,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 21:42:08,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 21:42:18,988] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 21:46:50,145] INFO : Step 81000/100000 rank4 | Train Loss: 11.7500 | Train Loss Unweighted: 3.6065 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.8934 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2712 | Regression_eta: 0.1074 | Regression_pt: 0.2244 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4130 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5159 | Classification_binary: 2.9734 | Regression_cos_phi: 1.7763 | Regression_energy: 1.4574 | Regression_eta: 1.5943 | Regression_pt: 1.6093 | Regression_sin_phi: 1.9543 | LR: 7.11e-05 | DataLoad Time: 0.0021s | Model Forward Time: 8.6906s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0975 | max_forward: 0.3237 | max_loss: 0.0036 | max_backward: 0.5496 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 6.3197 | max_tensorboard_flush: 0.0011 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6963.5 | uss_MB: 4835.8 | child_rss_MB: 8415.8 | allocated_MB: 619.9 | reserved_MB: 28598.0 | inactive_split_MB: 1192.4 +rank4 [2026-10-27 21:50:33,876] INFO : Step 82000/100000 rank4 | Train Loss: 13.6704 | Train Loss Unweighted: 4.5604 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0419 | Regression_energy: 0.4618 | Regression_eta: 0.0599 | Regression_pt: 0.4291 | Regression_sin_phi: 0.0407 | Group Train Loss Unweighted (window mean): physical_pu: 3.6250 | physical_nopu: 4.4152 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4278 | Classification_binary: 3.6340 | Regression_cos_phi: 1.5856 | Regression_energy: 1.6088 | Regression_eta: 0.7352 | Regression_pt: 1.8385 | Regression_sin_phi: 1.8195 | LR: 6.80e-05 | DataLoad Time: 0.0026s | Model Forward Time: 3.1344s | Phase Times: max_pre_step_barrier: 0.1835 | max_batch_to_device: 0.0874 | max_forward: 0.1976 | max_loss: 0.0036 | max_backward: 0.1170 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 1.6260 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 5659.6 | uss_MB: 4976.9 | child_rss_MB: 8356.3 | allocated_MB: 607.5 | reserved_MB: 28598.0 | inactive_split_MB: 253.8 +rank4 [2026-10-27 21:54:19,227] INFO : Step 83000/100000 rank4 | Train Loss: 12.0270 | Train Loss Unweighted: 4.2414 | Train Loss Components: Classification: 0.0092 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0499 | Regression_energy: 0.3056 | Regression_eta: 0.0671 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4141 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2870 | Classification_binary: 3.4698 | Regression_cos_phi: 1.7028 | Regression_energy: 1.5645 | Regression_eta: 1.0463 | Regression_pt: 1.7303 | Regression_sin_phi: 1.6505 | LR: 6.51e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.7069s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1966 | max_loss: 0.0138 | max_backward: 0.1124 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 2.4026 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 5659.6 | uss_MB: 4977.1 | child_rss_MB: 8350.2 | allocated_MB: 609.6 | reserved_MB: 28598.0 | inactive_split_MB: 289.5 +rank4 [2026-10-27 21:58:08,515] INFO : Step 84000/100000 rank4 | Train Loss: 12.5172 | Train Loss Unweighted: 4.2412 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.4173 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3464 | Regression_eta: 0.0637 | Regression_pt: 0.3052 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4149 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5219 | Classification_binary: 3.3994 | Regression_cos_phi: 1.7104 | Regression_energy: 1.6086 | Regression_eta: 1.1177 | Regression_pt: 1.8426 | Regression_sin_phi: 1.6792 | LR: 6.23e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.2646s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1103 | max_loss: 0.0037 | max_backward: 0.1159 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 2.0319 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5655.6 | uss_MB: 4977.8 | child_rss_MB: 8312.0 | allocated_MB: 605.9 | reserved_MB: 28598.0 | inactive_split_MB: 208.8 +rank4 [2026-10-27 22:01:46,561] INFO : Step 85000/100000 rank4 | Train Loss: 15.2814 | Train Loss Unweighted: 4.6863 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.3811 | Regression_cos_phi: 0.0432 | Regression_energy: 0.5842 | Regression_eta: 0.0608 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6173 | physical_nopu: 4.4078 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5255 | Classification_binary: 3.1279 | Regression_cos_phi: 1.4756 | Regression_energy: 4.1372 | Regression_eta: 0.7125 | Regression_pt: 5.3803 | Regression_sin_phi: 1.9422 | LR: 5.96e-05 | DataLoad Time: 0.0021s | Model Forward Time: 3.0839s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.0993 | max_loss: 0.0558 | max_backward: 0.1762 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 1.8022 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 5645.3 | uss_MB: 4977.4 | child_rss_MB: 8242.1 | allocated_MB: 606.6 | reserved_MB: 28598.0 | inactive_split_MB: 257.4 +rank4 [2026-10-27 22:05:28,665] INFO : Step 86000/100000 rank4 | Train Loss: 12.8074 | Train Loss Unweighted: 4.3373 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.5785 | Regression_cos_phi: 0.0543 | Regression_energy: 0.3116 | Regression_eta: 0.0747 | Regression_pt: 0.2515 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.6255 | physical_nopu: 4.4086 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2971 | Classification_binary: 3.6633 | Regression_cos_phi: 1.9320 | Regression_energy: 1.7549 | Regression_eta: 1.0552 | Regression_pt: 1.7971 | Regression_sin_phi: 2.7484 | LR: 5.71e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.0433s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1215 | max_loss: 0.0037 | max_backward: 0.1229 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 1.6761 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5636.3 | uss_MB: 4977.9 | child_rss_MB: 8180.8 | allocated_MB: 608.3 | reserved_MB: 29688.0 | inactive_split_MB: 271.8 +rank4 [2026-10-27 22:09:22,620] INFO : Step 87000/100000 rank4 | Train Loss: 12.9093 | Train Loss Unweighted: 4.3003 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.3807 | Regression_cos_phi: 0.0447 | Regression_energy: 0.4014 | Regression_eta: 0.0615 | Regression_pt: 0.3523 | Regression_sin_phi: 0.0424 | Group Train Loss Unweighted (window mean): physical_pu: 3.6211 | physical_nopu: 4.4120 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5009 | Classification_binary: 3.4475 | Regression_cos_phi: 1.5593 | Regression_energy: 1.8405 | Regression_eta: 0.8305 | Regression_pt: 2.1477 | Regression_sin_phi: 1.7547 | LR: 5.48e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.0851s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0203 | max_forward: 0.1299 | max_loss: 0.0038 | max_backward: 0.1175 | max_optimizer_step: 0.2192 | max_tensorboard_logging: 1.8010 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5636.1 | uss_MB: 4575.9 | child_rss_MB: 8182.7 | allocated_MB: 607.0 | reserved_MB: 29688.0 | inactive_split_MB: 231.0 +rank4 [2026-10-27 22:13:02,799] INFO : Step 88000/100000 rank4 | Train Loss: 15.2369 | Train Loss Unweighted: 4.6456 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4525 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5240 | Regression_eta: 0.0665 | Regression_pt: 0.4840 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6192 | physical_nopu: 4.4119 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5381 | Classification_binary: 3.6652 | Regression_cos_phi: 1.9867 | Regression_energy: 3.2686 | Regression_eta: 1.4383 | Regression_pt: 4.1295 | Regression_sin_phi: 2.1148 | LR: 5.26e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.1173s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0007 | max_forward: 0.1127 | max_loss: 0.0236 | max_backward: 0.1542 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 1.7956 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5636.1 | uss_MB: 4977.3 | child_rss_MB: 8181.7 | allocated_MB: 609.0 | reserved_MB: 29688.0 | inactive_split_MB: 221.0 +rank4 [2026-10-27 22:16:52,919] INFO : Step 89000/100000 rank4 | Train Loss: 12.8609 | Train Loss Unweighted: 4.2765 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.4665 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3343 | Regression_eta: 0.0681 | Regression_pt: 0.2896 | Regression_sin_phi: 0.0555 | Group Train Loss Unweighted (window mean): physical_pu: 3.6177 | physical_nopu: 4.4110 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4014 | Classification_binary: 3.5839 | Regression_cos_phi: 1.7509 | Regression_energy: 1.9521 | Regression_eta: 1.6004 | Regression_pt: 2.1237 | Regression_sin_phi: 2.4818 | LR: 5.06e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.1954s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1194 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 1.9452 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 5632.5 | uss_MB: 4977.7 | child_rss_MB: 8137.7 | allocated_MB: 612.6 | reserved_MB: 30628.0 | inactive_split_MB: 309.7 +rank4 [2026-10-27 22:20:43,711] INFO : Step 90000/100000 rank4 | Train Loss: 11.6102 | Train Loss Unweighted: 3.6127 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9177 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2661 | Regression_eta: 0.1040 | Regression_pt: 0.2155 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4109 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4213 | Classification_binary: 2.9308 | Regression_cos_phi: 1.7449 | Regression_energy: 1.3906 | Regression_eta: 1.5721 | Regression_pt: 1.4931 | Regression_sin_phi: 1.9197 | LR: 4.88e-05 | DataLoad Time: 0.0021s | Model Forward Time: 3.5773s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0110 | max_forward: 0.1856 | max_loss: 0.0145 | max_backward: 0.4526 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 1.8620 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 5624.5 | uss_MB: 4985.7 | child_rss_MB: 8062.4 | allocated_MB: 617.2 | reserved_MB: 30628.0 | inactive_split_MB: 962.2 +rank4 [2026-10-27 22:20:44,690] INFO : Running validation on rank4 +rank4 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 22:22:45,205] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 22:22:45,606] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 22:23:10,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 22:23:10,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 22:23:11,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 22:23:12,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 22:23:12,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 22:23:13,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 22:23:13,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 22:23:14,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 22:23:24,054] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 22:23:24,365] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 22:23:24,733] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 22:23:39,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 22:23:40,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 22:23:40,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 22:23:41,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 22:23:41,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 22:23:42,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 22:23:42,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 22:23:43,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 22:23:53,476] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-27 22:23:53,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 22:23:54,273] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 22:24:09,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 22:24:10,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 22:24:10,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 22:24:11,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 22:24:11,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 22:24:12,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 22:24:12,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 22:24:13,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 22:24:23,143] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 22:28:41,211] INFO : Step 91000/100000 rank4 | Train Loss: 12.9063 | Train Loss Unweighted: 4.3371 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5236 | Regression_cos_phi: 0.0531 | Regression_energy: 0.3431 | Regression_eta: 0.0707 | Regression_pt: 0.2829 | Regression_sin_phi: 0.0512 | Group Train Loss Unweighted (window mean): physical_pu: 3.6180 | physical_nopu: 4.4087 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3628 | Classification_binary: 3.5379 | Regression_cos_phi: 2.2940 | Regression_energy: 1.7235 | Regression_eta: 0.9118 | Regression_pt: 1.8677 | Regression_sin_phi: 2.4695 | LR: 4.71e-05 | DataLoad Time: 0.0019s | Model Forward Time: 4.9000s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0055 | max_forward: 0.1521 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 3.4850 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7066.6 | uss_MB: 6432.9 | child_rss_MB: 8020.4 | allocated_MB: 607.8 | reserved_MB: 28796.0 | inactive_split_MB: 320.2 +rank4 [2026-10-27 22:32:23,431] INFO : Step 92000/100000 rank4 | Train Loss: 15.4341 | Train Loss Unweighted: 4.7247 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4285 | Regression_cos_phi: 0.0457 | Regression_energy: 0.5811 | Regression_eta: 0.0735 | Regression_pt: 0.5375 | Regression_sin_phi: 0.0425 | Group Train Loss Unweighted (window mean): physical_pu: 3.6150 | physical_nopu: 4.4032 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5540 | Classification_binary: 3.5008 | Regression_cos_phi: 1.8533 | Regression_energy: 3.3912 | Regression_eta: 1.1904 | Regression_pt: 4.3072 | Regression_sin_phi: 2.6071 | LR: 4.57e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.3951s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.2696 | max_loss: 0.0037 | max_backward: 0.1715 | max_optimizer_step: 0.3237 | max_tensorboard_logging: 1.7768 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7051.6 | uss_MB: 6432.9 | child_rss_MB: 7931.4 | allocated_MB: 609.3 | reserved_MB: 28796.0 | inactive_split_MB: 222.8 +rank4 [2026-10-27 22:36:08,796] INFO : Step 93000/100000 rank4 | Train Loss: 13.9744 | Train Loss Unweighted: 4.4854 | Train Loss Components: Classification: 0.0118 | Classification_binary: 3.5124 | Regression_cos_phi: 0.0578 | Regression_energy: 0.4245 | Regression_eta: 0.0652 | Regression_pt: 0.3624 | Regression_sin_phi: 0.0513 | Group Train Loss Unweighted (window mean): physical_pu: 3.6205 | physical_nopu: 4.4075 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3427 | Classification_binary: 3.3180 | Regression_cos_phi: 2.3209 | Regression_energy: 2.1397 | Regression_eta: 0.7447 | Regression_pt: 2.3305 | Regression_sin_phi: 1.9622 | LR: 4.43e-05 | DataLoad Time: 0.0024s | Model Forward Time: 2.9848s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1098 | max_loss: 0.0036 | max_backward: 0.1173 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 1.7709 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5706.9 | uss_MB: 5092.9 | child_rss_MB: 7894.0 | allocated_MB: 606.4 | reserved_MB: 30394.0 | inactive_split_MB: 244.2 +rank4 [2026-10-27 22:39:53,036] INFO : Step 94000/100000 rank4 | Train Loss: 12.4397 | Train Loss Unweighted: 4.2538 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4230 | Regression_cos_phi: 0.0415 | Regression_energy: 0.3513 | Regression_eta: 0.0702 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6164 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4705 | Classification_binary: 3.3467 | Regression_cos_phi: 1.4271 | Regression_energy: 1.7328 | Regression_eta: 0.7235 | Regression_pt: 2.1295 | Regression_sin_phi: 1.5541 | LR: 4.32e-05 | DataLoad Time: 0.0011s | Model Forward Time: 3.1524s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0009 | max_forward: 0.1540 | max_loss: 0.0036 | max_backward: 0.2309 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 1.7723 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 5695.9 | uss_MB: 4753.0 | child_rss_MB: 7806.9 | allocated_MB: 607.1 | reserved_MB: 30394.0 | inactive_split_MB: 232.2 +rank4 [2026-10-27 22:43:36,954] INFO : Step 95000/100000 rank4 | Train Loss: 14.9177 | Train Loss Unweighted: 4.6108 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3911 | Regression_cos_phi: 0.0436 | Regression_energy: 0.5612 | Regression_eta: 0.0598 | Regression_pt: 0.4905 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6121 | physical_nopu: 4.4076 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4786 | Classification_binary: 3.3974 | Regression_cos_phi: 2.1364 | Regression_energy: 3.0289 | Regression_eta: 1.0431 | Regression_pt: 3.6819 | Regression_sin_phi: 1.6951 | LR: 4.22e-05 | DataLoad Time: 0.0009s | Model Forward Time: 3.0363s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1111 | max_loss: 0.0199 | max_backward: 0.1395 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.7978 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5695.9 | uss_MB: 5093.1 | child_rss_MB: 7811.5 | allocated_MB: 609.0 | reserved_MB: 30394.0 | inactive_split_MB: 285.3 +rank4 [2026-10-27 22:47:14,355] INFO : Step 96000/100000 rank4 | Train Loss: 12.4738 | Train Loss Unweighted: 4.2244 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.4884 | Regression_cos_phi: 0.0540 | Regression_energy: 0.2967 | Regression_eta: 0.0804 | Regression_pt: 0.2429 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6137 | physical_nopu: 4.4081 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2830 | Classification_binary: 3.4973 | Regression_cos_phi: 1.7266 | Regression_energy: 1.3804 | Regression_eta: 0.9386 | Regression_pt: 1.5193 | Regression_sin_phi: 1.9960 | LR: 4.14e-05 | DataLoad Time: 0.0030s | Model Forward Time: 3.1598s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0007 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.2003 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 1.8675 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5695.9 | uss_MB: 5093.2 | child_rss_MB: 7804.9 | allocated_MB: 608.1 | reserved_MB: 30394.0 | inactive_split_MB: 259.4 +rank4 [2026-10-27 22:50:52,983] INFO : Step 97000/100000 rank4 | Train Loss: 12.7922 | Train Loss Unweighted: 4.2852 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.4355 | Regression_cos_phi: 0.0455 | Regression_energy: 0.3604 | Regression_eta: 0.0703 | Regression_pt: 0.3091 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6136 | physical_nopu: 4.4015 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5966 | Classification_binary: 3.2875 | Regression_cos_phi: 1.8005 | Regression_energy: 1.9311 | Regression_eta: 1.0257 | Regression_pt: 2.2341 | Regression_sin_phi: 1.8243 | LR: 4.08e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.0107s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1110 | max_loss: 0.0174 | max_backward: 0.1239 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 1.7282 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0908 | Memory: rss_MB: 5687.4 | uss_MB: 5092.4 | child_rss_MB: 7746.9 | allocated_MB: 607.2 | reserved_MB: 30394.0 | inactive_split_MB: 265.3 +rank4 [2026-10-27 22:54:49,600] INFO : Step 98000/100000 rank4 | Train Loss: 13.8508 | Train Loss Unweighted: 4.4512 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3736 | Regression_cos_phi: 0.0443 | Regression_energy: 0.4792 | Regression_eta: 0.0606 | Regression_pt: 0.4354 | Regression_sin_phi: 0.0433 | Group Train Loss Unweighted (window mean): physical_pu: 3.6169 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5173 | Classification_binary: 3.4257 | Regression_cos_phi: 2.1156 | Regression_energy: 2.6232 | Regression_eta: 1.3027 | Regression_pt: 2.8607 | Regression_sin_phi: 1.8268 | LR: 4.04e-05 | DataLoad Time: 0.0029s | Model Forward Time: 3.0649s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0119 | max_forward: 0.1317 | max_loss: 0.0036 | max_backward: 0.1246 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.7312 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0878 | Memory: rss_MB: 5687.4 | uss_MB: 5093.6 | child_rss_MB: 7731.7 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 292.7 +rank4 [2026-10-27 22:58:30,099] INFO : Step 99000/100000 rank4 | Train Loss: 12.5729 | Train Loss Unweighted: 4.3128 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5536 | Regression_cos_phi: 0.0519 | Regression_energy: 0.3097 | Regression_eta: 0.0644 | Regression_pt: 0.2679 | Regression_sin_phi: 0.0529 | Group Train Loss Unweighted (window mean): physical_pu: 3.6154 | physical_nopu: 4.4035 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3191 | Classification_binary: 3.6223 | Regression_cos_phi: 2.0415 | Regression_energy: 1.7186 | Regression_eta: 1.0584 | Regression_pt: 1.9510 | Regression_sin_phi: 2.1800 | LR: 4.01e-05 | DataLoad Time: 0.0115s | Model Forward Time: 3.0441s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0008 | max_forward: 0.1158 | max_loss: 0.0113 | max_backward: 0.1168 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 1.7726 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 5687.4 | uss_MB: 5093.5 | child_rss_MB: 7727.1 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 283.7 +rank4 [2026-10-27 23:02:17,099] INFO : Step 100000/100000 rank4 | Train Loss: 11.9488 | Train Loss Unweighted: 3.6450 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9057 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2836 | Regression_eta: 0.1038 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6201 | physical_nopu: 4.4070 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4701 | Classification_binary: 2.9952 | Regression_cos_phi: 1.9246 | Regression_energy: 1.5145 | Regression_eta: 1.5428 | Regression_pt: 1.7373 | Regression_sin_phi: 1.9999 | LR: 4.00e-05 | DataLoad Time: 0.0031s | Model Forward Time: 6.3804s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0031 | max_forward: 0.2860 | max_loss: 0.0036 | max_backward: 0.4897 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 4.4601 | max_tensorboard_flush: 0.0009 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5687.2 | uss_MB: 4556.1 | child_rss_MB: 7711.3 | allocated_MB: 618.6 | reserved_MB: 30394.0 | inactive_split_MB: 1112.1 +rank4 [2026-10-27 23:02:18,080] INFO : Running validation on rank4 +rank4 [2026-10-27 23:02:18,080] INFO : ResumableSampler physical_pu:valid resetting +rank4 [2026-10-27 23:02:18,080] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank4 [2026-10-27 23:02:18,080] INFO : ResumableSampler physical_nopu:valid resetting +rank4 [2026-10-27 23:02:18,080] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank4 [2026-10-27 23:04:23,625] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 23:04:24,087] INFO : Running predictions on cms_pf_ttbar +rank4 [2026-10-27 23:04:49,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_4_0.parquet +rank4 [2026-10-27 23:04:50,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_4_1.parquet +rank4 [2026-10-27 23:04:50,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_4_2.parquet +rank4 [2026-10-27 23:04:51,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_4_3.parquet +rank4 [2026-10-27 23:04:51,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_4_4.parquet +rank4 [2026-10-27 23:04:52,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_4_5.parquet +rank4 [2026-10-27 23:04:52,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_4_6.parquet +rank4 [2026-10-27 23:04:53,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_4_7.parquet +rank4 [2026-10-27 23:05:03,219] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 23:05:03,812] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 23:05:04,178] INFO : Running predictions on cms_pf_qcd +rank4 [2026-10-27 23:05:19,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_4_0.parquet +rank4 [2026-10-27 23:05:20,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_4_1.parquet +rank4 [2026-10-27 23:05:20,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_4_2.parquet +rank4 [2026-10-27 23:05:21,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_4_3.parquet +rank4 [2026-10-27 23:05:21,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_4_4.parquet +rank4 [2026-10-27 23:05:22,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_4_5.parquet +rank4 [2026-10-27 23:05:22,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_4_6.parquet +rank4 [2026-10-27 23:05:23,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_4_7.parquet +rank4 [2026-10-27 23:05:33,199] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-27 23:05:33,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-27 23:05:34,200] INFO : Running predictions on cms_pf_ztt +rank4 [2026-10-27 23:05:49,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_4_0.parquet +rank4 [2026-10-27 23:05:49,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_4_1.parquet +rank4 [2026-10-27 23:05:50,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_4_2.parquet +rank4 [2026-10-27 23:05:51,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_4_3.parquet +rank4 [2026-10-27 23:05:51,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_4_4.parquet +rank4 [2026-10-27 23:05:52,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_4_5.parquet +rank4 [2026-10-27 23:05:52,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_4_6.parquet +rank4 [2026-10-27 23:05:53,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_4_7.parquet +rank4 [2026-10-27 23:06:03,170] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-27 23:06:03,951] INFO : Training completed. Total time on device 4: 414.794min diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.5 b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.5 new file mode 100644 index 0000000000000000000000000000000000000000..740102f04032ef967ca9a4dc51e77eb6ff51dd37 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.5 @@ -0,0 +1,897 @@ +rank5 [2026-10-27 16:10:10,404] INFO : Initializing model with seed=12345; process seed=12350 +rank5 [2026-10-27 16:10:10,405] INFO : configured dtype=torch.bfloat16 for autocast +rank5 [2026-10-27 16:10:35,267] INFO : Instantiating model +rank5 [2026-10-27 16:10:35,267] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank5 [2026-10-27 16:10:35,267] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank5 [2026-10-27 16:10:35,295] INFO : Input encoding initialization took 0.03s +rank5 [2026-10-27 16:10:35,296] INFO : nn0 parameters: 2130944 +rank5 [2026-10-27 16:10:35,296] INFO : Initializing shared backbone layers, num_convs=3 +rank5 [2026-10-27 16:10:35,296] INFO : Initializing backbone layer 0 +rank5 [2026-10-27 16:10:35,350] INFO : Initializing backbone layer 1 +rank5 [2026-10-27 16:10:35,379] INFO : Initializing backbone layer 2 +rank5 [2026-10-27 16:10:35,407] INFO : Backbone initialization took 0.11s +rank5 [2026-10-27 16:10:35,407] INFO : backbone parameters: 15765696 +rank5 [2026-10-27 16:10:35,408] INFO : Initializing output DNNs +rank5 [2026-10-27 16:10:35,441] INFO : Output DNNs initialization took 0.03s +rank5 [2026-10-27 16:10:35,441] INFO : backbone_mode=BackboneMode.SHARED +rank5 [2026-10-27 16:10:35,441] INFO : jagged_attention enabled=False +rank5 [2026-10-27 16:10:35,441] INFO : flash_attn_varlen enabled=False +rank5 [2026-10-27 16:10:35,441] INFO : task_queries enabled=False +rank5 [2026-10-27 16:10:35,441] INFO : classification_norm parameters: 0 +rank5 [2026-10-27 16:10:35,441] INFO : regression_norm parameters: 0 +rank5 [2026-10-27 16:10:35,441] INFO : classification_query parameters: 0 +rank5 [2026-10-27 16:10:35,441] INFO : regression_query parameters: 0 +rank5 [2026-10-27 16:10:35,441] INFO : classification_readout parameters: 0 +rank5 [2026-10-27 16:10:35,441] INFO : regression_readout parameters: 0 +rank5 [2026-10-27 16:10:35,441] INFO : output_mode=elementwise +rank5 [2026-10-27 16:10:35,441] INFO : set_decoder parameters: 0 +rank5 [2026-10-27 16:10:35,441] INFO : nn_binary_particle parameters: 264706 +rank5 [2026-10-27 16:10:35,442] INFO : nn_pid parameters: 268297 +rank5 [2026-10-27 16:10:35,442] INFO : nn_pt parameters: 2113544 +rank5 [2026-10-27 16:10:35,442] INFO : nn_eta parameters: 264706 +rank5 [2026-10-27 16:10:35,442] INFO : nn_sin_phi parameters: 264706 +rank5 [2026-10-27 16:10:35,442] INFO : nn_cos_phi parameters: 264706 +rank5 [2026-10-27 16:10:35,442] INFO : nn_energy parameters: 2113544 +rank5 [2026-10-27 16:10:35,443] INFO : Total MLPF parameters: 23450849 +rank5 [2026-10-27 16:10:35,443] INFO : MLPF __init__ done +rank5 [2026-10-27 16:10:35,443] INFO : Instantiated model +rank5 [2026-10-27 16:10:35,443] INFO : Moving model to device rank=5 +rank5 [2026-10-27 16:10:35,468] INFO : Moved model to device rank=5 +rank5 [2026-10-27 16:10:35,469] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank5 [2026-10-27 16:10:35,470] INFO : Configured model for SyncBatchNorm rank=5 +rank5 [2026-10-27 16:10:36,003] INFO : Configured model for DistributedDataParallel rank=5 +rank5 [2026-10-27 16:10:36,123] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +202 module.nn_energy.nn.7.0.bias ... 0 +203 module.nn_energy.nn.7.2.weight ... 0 +204 module.nn_energy.nn.7.2.bias ... 0 +205 module.nn_energy.nn.7.4.weight ... 0 +206 module.nn_energy.nn.7.4.bias ... 0 + +[207 rows x 3 columns] +rank5 [2026-10-27 16:10:36,123] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x HEPTv2Layer( + embedding_dim=512, num_heads=16, width=512, dropout=0.0, block_size=100, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=512, out_features=512, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=512, out_features=512, bias=False) + (w_k): Linear(in_features=512, out_features=512, bias=False) + (w_v): Linear(in_features=512, out_features=512, bias=False) + (w_rpe): Linear(in_features=10, out_features=512, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=2048, bias=True) + (1): LayerNorm((2048,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=2048, out_features=512, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=512, out_features=2048, bias=False) + (up_proj): Linear(in_features=512, out_features=2048, bias=False) + (down_proj): Linear(in_features=2048, out_features=512, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank5 [2026-10-27 16:10:36,123] INFO : Trainable parameters: 23450849 +rank5 [2026-10-27 16:10:36,124] INFO : Non-trainable parameters: 22386 +rank5 [2026-10-27 16:10:36,124] INFO : Total parameters: 23473235 +rank5 [2026-10-27 16:10:36,126] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 262144 0 + module.backbone.0.attn.out_linear.bias 512 0 + module.backbone.0.attn.e2lsh.alpha 0 1632 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 262144 0 + module.backbone.0.w_k.weight 262144 0 + module.backbone.0.w_v.weight 262144 0 + module.backbone.0.w_rpe.weight 0 5120 + module.backbone.0.w_rpe.bias 0 512 +module.backbone.0.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm2.weight 512 0 + module.backbone.0.q_norm.weight 32 0 + module.backbone.0.k_norm.weight 32 0 + module.backbone.0.ff.gate_proj.weight 1048576 0 + module.backbone.0.ff.up_proj.weight 1048576 0 + module.backbone.0.ff.down_proj.weight 1048576 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 262144 0 + module.backbone.1.attn.out_linear.bias 512 0 + module.backbone.1.attn.e2lsh.alpha 0 1632 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 262144 0 + module.backbone.1.w_k.weight 262144 0 + module.backbone.1.w_v.weight 262144 0 + module.backbone.1.w_rpe.weight 0 5120 + module.backbone.1.w_rpe.bias 0 512 +module.backbone.1.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm2.weight 512 0 + module.backbone.1.q_norm.weight 32 0 + module.backbone.1.k_norm.weight 32 0 + module.backbone.1.ff.gate_proj.weight 1048576 0 + module.backbone.1.ff.up_proj.weight 1048576 0 + module.backbone.1.ff.down_proj.weight 1048576 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 262144 0 + module.backbone.2.attn.out_linear.bias 512 0 + module.backbone.2.attn.e2lsh.alpha 0 1632 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 262144 0 + module.backbone.2.w_k.weight 262144 0 + module.backbone.2.w_v.weight 262144 0 + module.backbone.2.w_rpe.weight 0 5120 + module.backbone.2.w_rpe.bias 0 512 +module.backbone.2.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm2.weight 512 0 + module.backbone.2.q_norm.weight 32 0 + module.backbone.2.k_norm.weight 32 0 + module.backbone.2.ff.gate_proj.weight 1048576 0 + module.backbone.2.ff.up_proj.weight 1048576 0 + module.backbone.2.ff.down_proj.weight 1048576 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank5 [2026-10-27 16:10:36,126] INFO : Getting dataloaders +rank5 [2026-10-27 16:10:36,126] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:43,312] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:44,206] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:44,830] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank5 [2026-10-27 16:10:44,830] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:46,060] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:46,991] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:47,843] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank5 [2026-10-27 16:10:47,844] INFO : Creating WeightedMixIterator with 2 data loaders. +rank5 [2026-10-27 16:10:47,844] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:47,994] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:48,143] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:48,288] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank5 [2026-10-27 16:10:48,288] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:48,448] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:48,616] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:10:48,768] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank5 [2026-10-27 16:10:48,768] INFO : Creating InterleavedIterator with 2 data loaders. +rank5 [2026-10-27 16:11:16,502] INFO : Got dataloaders +rank5 [2026-10-27 16:11:16,503] INFO : loader split=train rank=5 len=232058 +rank5 [2026-10-27 16:11:16,503] INFO : loader split=valid rank=5 len=48 +rank5 [2026-10-27 16:11:16,503] INFO : Starting training from step 1 to 100000 on rank 5 of 8 +rank5 [2026-10-27 16:11:16,503] INFO : Creating train_iterator +rank5 [2026-10-27 16:11:16,503] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank5 [2026-10-27 16:11:30,801] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank5 [2026-10-27 16:11:45,801] INFO : Created train_iterator +rank5 [2026-10-27 16:12:32,259] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.129867712855339, 'Classification': 0.16580097352620216, 'Regression_pt': 0.7168090478330851, 'Regression_eta': 0.335200086296536, 'Regression_sin_phi': 0.1274821354681626, 'Regression_cos_phi': 0.1371471309568733, 'Regression_energy': 0.9554258581250906}, weights={'Classification_binary': 1.0, 'Classification': 30.939912796020508, 'Regression_pt': 7.156533241271973, 'Regression_eta': 15.3038969039917, 'Regression_sin_phi': 40.23989486694336, 'Regression_cos_phi': 37.40412139892578, 'Regression_energy': 5.369194984436035} +rank5 [2026-10-27 16:15:18,321] INFO : Step 1000/100000 rank5 | Train Loss: 16.8632 | Train Loss Unweighted: 5.4934 | Train Loss Components: Classification: 0.0189 | Classification_binary: 4.1833 | Regression_cos_phi: 0.0509 | Regression_energy: 0.5806 | Regression_eta: 0.0776 | Regression_pt: 0.5302 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 5.1301 | physical_nopu: 6.0772 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5171 | Classification_binary: 4.2595 | Regression_cos_phi: 1.5862 | Regression_energy: 2.7860 | Regression_eta: 1.2925 | Regression_pt: 3.5156 | Regression_sin_phi: 1.8313 | LR: 4.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.3138s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0014 | max_forward: 0.0387 | max_loss: 0.0036 | max_backward: 0.0664 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 1.2817 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 4374.6 | uss_MB: 3102.3 | child_rss_MB: 11636.1 | allocated_MB: 609.1 | reserved_MB: 27788.0 | inactive_split_MB: 244.1 +rank5 [2026-10-27 16:18:42,006] INFO : Step 2000/100000 rank5 | Train Loss: 17.7204 | Train Loss Unweighted: 5.4242 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.9240 | Regression_cos_phi: 0.0505 | Regression_energy: 0.6758 | Regression_eta: 0.0665 | Regression_pt: 0.6366 | Regression_sin_phi: 0.0551 | Group Train Loss Unweighted (window mean): physical_pu: 4.2802 | physical_nopu: 5.3470 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4097 | Classification_binary: 3.9801 | Regression_cos_phi: 1.8682 | Regression_energy: 3.0193 | Regression_eta: 0.9568 | Regression_pt: 3.7444 | Regression_sin_phi: 2.8039 | LR: 4.00e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8063s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1422 | max_loss: 0.0036 | max_backward: 0.1717 | max_optimizer_step: 0.1501 | max_tensorboard_logging: 0.7287 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 4367.5 | uss_MB: 3102.8 | child_rss_MB: 11637.6 | allocated_MB: 607.0 | reserved_MB: 27788.0 | inactive_split_MB: 326.9 +rank5 [2026-10-27 16:22:10,507] INFO : Step 3000/100000 rank5 | Train Loss: 14.3102 | Train Loss Unweighted: 4.8875 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.8970 | Regression_cos_phi: 0.0491 | Regression_energy: 0.4277 | Regression_eta: 0.0716 | Regression_pt: 0.3769 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 4.0705 | physical_nopu: 5.1319 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5154 | Classification_binary: 3.7674 | Regression_cos_phi: 1.7683 | Regression_energy: 2.1357 | Regression_eta: 0.9272 | Regression_pt: 2.5718 | Regression_sin_phi: 1.9672 | LR: 3.99e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.0067s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0782 | max_loss: 0.0037 | max_backward: 0.0505 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 1.0271 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4295.2 | uss_MB: 3034.9 | child_rss_MB: 11625.5 | allocated_MB: 607.6 | reserved_MB: 28576.0 | inactive_split_MB: 261.0 +rank5 [2026-10-27 16:25:44,194] INFO : Step 4000/100000 rank5 | Train Loss: 12.6160 | Train Loss Unweighted: 3.9080 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.1145 | Regression_cos_phi: 0.0484 | Regression_energy: 0.3088 | Regression_eta: 0.1117 | Regression_pt: 0.2597 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.9908 | physical_nopu: 5.0219 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5291 | Classification_binary: 3.1296 | Regression_cos_phi: 1.7886 | Regression_energy: 1.7504 | Regression_eta: 1.7919 | Regression_pt: 1.9679 | Regression_sin_phi: 2.0423 | LR: 3.99e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4272s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0032 | max_forward: 0.1365 | max_loss: 0.0036 | max_backward: 0.2496 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 1.0566 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 4292.4 | uss_MB: 3043.1 | child_rss_MB: 11491.5 | allocated_MB: 617.3 | reserved_MB: 28576.0 | inactive_split_MB: 1049.1 +rank5 [2026-10-27 16:29:29,026] INFO : Step 5000/100000 rank5 | Train Loss: 13.7309 | Train Loss Unweighted: 4.7575 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.8762 | Regression_cos_phi: 0.0508 | Regression_energy: 0.3661 | Regression_eta: 0.0922 | Regression_pt: 0.3105 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.9453 | physical_nopu: 4.9561 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4404 | Classification_binary: 3.8289 | Regression_cos_phi: 1.9744 | Regression_energy: 1.8154 | Regression_eta: 1.4750 | Regression_pt: 2.0856 | Regression_sin_phi: 1.8737 | LR: 3.98e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.0871s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.0008 | max_forward: 0.0371 | max_loss: 0.0036 | max_backward: 0.0568 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 2.1174 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4241.6 | uss_MB: 3036.2 | child_rss_MB: 11146.9 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 248.7 +rank5 [2026-10-27 16:33:03,739] INFO : Step 6000/100000 rank5 | Train Loss: 15.2316 | Train Loss Unweighted: 4.9080 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.7479 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5147 | Regression_eta: 0.0679 | Regression_pt: 0.4609 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.9059 | physical_nopu: 4.9043 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4604 | Classification_binary: 3.7999 | Regression_cos_phi: 2.0421 | Regression_energy: 2.4562 | Regression_eta: 0.8644 | Regression_pt: 2.9897 | Regression_sin_phi: 1.7575 | LR: 3.97e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.0044s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0421 | max_loss: 0.0036 | max_backward: 0.0620 | max_optimizer_step: 0.1489 | max_tensorboard_logging: 2.0592 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4216.9 | uss_MB: 3036.3 | child_rss_MB: 11044.6 | allocated_MB: 610.3 | reserved_MB: 28576.0 | inactive_split_MB: 254.0 +rank5 [2026-10-27 16:36:49,903] INFO : Step 7000/100000 rank5 | Train Loss: 14.7215 | Train Loss Unweighted: 4.8917 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8534 | Regression_cos_phi: 0.0503 | Regression_energy: 0.4508 | Regression_eta: 0.0791 | Regression_pt: 0.3921 | Regression_sin_phi: 0.0547 | Group Train Loss Unweighted (window mean): physical_pu: 3.8763 | physical_nopu: 4.8627 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2458 | Classification_binary: 3.8002 | Regression_cos_phi: 2.1391 | Regression_energy: 1.7006 | Regression_eta: 1.2777 | Regression_pt: 1.6445 | Regression_sin_phi: 2.1741 | LR: 3.96e-04 | DataLoad Time: 0.0016s | Model Forward Time: 4.5447s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0348 | max_loss: 0.0036 | max_backward: 0.0535 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 3.5409 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4194.7 | uss_MB: 3033.5 | child_rss_MB: 10983.9 | allocated_MB: 609.0 | reserved_MB: 28576.0 | inactive_split_MB: 216.2 +rank5 [2026-10-27 16:40:34,139] INFO : Step 8000/100000 rank5 | Train Loss: 14.2560 | Train Loss Unweighted: 4.8042 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.8045 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4333 | Regression_eta: 0.0601 | Regression_pt: 0.3883 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.8518 | physical_nopu: 4.8294 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6030 | Classification_binary: 3.7077 | Regression_cos_phi: 2.0085 | Regression_energy: 1.8774 | Regression_eta: 0.8101 | Regression_pt: 2.2050 | Regression_sin_phi: 1.4407 | LR: 3.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 4.8860s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1342 | max_loss: 0.0036 | max_backward: 0.0500 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 3.8645 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4176.9 | uss_MB: 3036.4 | child_rss_MB: 10934.3 | allocated_MB: 606.6 | reserved_MB: 28576.0 | inactive_split_MB: 255.4 +rank5 [2026-10-27 16:44:13,306] INFO : Step 9000/100000 rank5 | Train Loss: 15.9274 | Train Loss Unweighted: 4.9361 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.6437 | Regression_cos_phi: 0.0472 | Regression_energy: 0.5701 | Regression_eta: 0.0733 | Regression_pt: 0.5360 | Regression_sin_phi: 0.0496 | Group Train Loss Unweighted (window mean): physical_pu: 3.8376 | physical_nopu: 4.8057 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5705 | Classification_binary: 3.6443 | Regression_cos_phi: 1.9181 | Regression_energy: 3.4884 | Regression_eta: 0.8183 | Regression_pt: 4.3225 | Regression_sin_phi: 2.1943 | LR: 3.93e-04 | DataLoad Time: 0.0026s | Model Forward Time: 3.2103s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1320 | max_loss: 0.0037 | max_backward: 0.0555 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 2.0797 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 4157.5 | uss_MB: 3035.9 | child_rss_MB: 10877.2 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 321.4 +rank5 [2026-10-27 16:48:05,432] INFO : Step 10000/100000 rank5 | Train Loss: 13.0514 | Train Loss Unweighted: 4.5682 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.7678 | Regression_cos_phi: 0.0520 | Regression_energy: 0.3372 | Regression_eta: 0.0736 | Regression_pt: 0.2745 | Regression_sin_phi: 0.0521 | Group Train Loss Unweighted (window mean): physical_pu: 3.8237 | physical_nopu: 4.7797 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3324 | Classification_binary: 3.7490 | Regression_cos_phi: 1.5643 | Regression_energy: 2.0974 | Regression_eta: 0.9316 | Regression_pt: 2.3223 | Regression_sin_phi: 2.1269 | LR: 3.91e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.2567s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.0379 | max_loss: 0.0040 | max_backward: 0.0556 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 2.2516 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4149.4 | uss_MB: 3036.1 | child_rss_MB: 10872.4 | allocated_MB: 607.9 | reserved_MB: 28576.0 | inactive_split_MB: 258.7 +rank5 [2026-10-27 16:48:06,335] INFO : Running validation on rank5 +rank5 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 16:50:06,286] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:50:06,771] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 16:50:34,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 16:50:35,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 16:50:36,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 16:50:36,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 16:50:37,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 16:50:37,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 16:50:38,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 16:50:38,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 16:50:48,727] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-27 16:50:48,969] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:50:49,336] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 16:51:05,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 16:51:05,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 16:51:06,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 16:51:06,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 16:51:07,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 16:51:07,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 16:51:08,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 16:51:08,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 16:51:18,727] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 16:51:18,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 16:51:19,154] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 16:51:34,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 16:51:35,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 16:51:35,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 16:51:36,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 16:51:36,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 16:51:37,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 16:51:37,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 16:51:38,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 16:51:48,411] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 16:55:58,693] INFO : Step 11000/100000 rank5 | Train Loss: 14.4095 | Train Loss Unweighted: 4.7539 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.7591 | Regression_cos_phi: 0.0522 | Regression_energy: 0.4228 | Regression_eta: 0.0709 | Regression_pt: 0.3796 | Regression_sin_phi: 0.0518 | Group Train Loss Unweighted (window mean): physical_pu: 3.8077 | physical_nopu: 4.7444 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5630 | Classification_binary: 3.7888 | Regression_cos_phi: 1.7048 | Regression_energy: 2.3547 | Regression_eta: 1.3170 | Regression_pt: 2.6332 | Regression_sin_phi: 2.0628 | LR: 3.89e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.2000s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1127 | max_loss: 0.0074 | max_backward: 0.1410 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 1.9404 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6580.9 | uss_MB: 5462.7 | child_rss_MB: 10900.1 | allocated_MB: 610.6 | reserved_MB: 27422.0 | inactive_split_MB: 253.4 +rank5 [2026-10-27 16:59:46,314] INFO : Step 12000/100000 rank5 | Train Loss: 16.4725 | Train Loss Unweighted: 5.0863 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.5958 | Regression_cos_phi: 0.0410 | Regression_energy: 0.6842 | Regression_eta: 0.0610 | Regression_pt: 0.6484 | Regression_sin_phi: 0.0394 | Group Train Loss Unweighted (window mean): physical_pu: 3.8013 | physical_nopu: 4.7308 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5097 | Classification_binary: 3.5089 | Regression_cos_phi: 1.1536 | Regression_energy: 3.3144 | Regression_eta: 0.6785 | Regression_pt: 4.2220 | Regression_sin_phi: 1.3708 | LR: 3.87e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.5374s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1292 | max_loss: 0.0037 | max_backward: 0.1951 | max_optimizer_step: 0.2940 | max_tensorboard_logging: 2.1160 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6552.9 | uss_MB: 5462.9 | child_rss_MB: 10800.4 | allocated_MB: 609.5 | reserved_MB: 27422.0 | inactive_split_MB: 347.2 +rank5 [2026-10-27 17:03:40,557] INFO : Step 13000/100000 rank5 | Train Loss: 12.9993 | Train Loss Unweighted: 4.5752 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.7655 | Regression_cos_phi: 0.0483 | Regression_energy: 0.3375 | Regression_eta: 0.0674 | Regression_pt: 0.2918 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7898 | physical_nopu: 4.7063 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3246 | Classification_binary: 3.7721 | Regression_cos_phi: 1.9051 | Regression_energy: 2.1706 | Regression_eta: 0.9868 | Regression_pt: 2.5481 | Regression_sin_phi: 2.0409 | LR: 3.85e-04 | DataLoad Time: 0.0088s | Model Forward Time: 3.0969s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1327 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.9573 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5436.1 | uss_MB: 4383.7 | child_rss_MB: 10499.2 | allocated_MB: 607.2 | reserved_MB: 28584.0 | inactive_split_MB: 340.8 +rank5 [2026-10-27 17:07:23,686] INFO : Step 14000/100000 rank5 | Train Loss: 12.1035 | Train Loss Unweighted: 3.7686 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.0394 | Regression_cos_phi: 0.0492 | Regression_energy: 0.2767 | Regression_eta: 0.1064 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0483 | Group Train Loss Unweighted (window mean): physical_pu: 3.7803 | physical_nopu: 4.6878 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4958 | Classification_binary: 3.0811 | Regression_cos_phi: 1.7907 | Regression_energy: 1.4811 | Regression_eta: 1.5164 | Regression_pt: 1.6800 | Regression_sin_phi: 2.0556 | LR: 3.83e-04 | DataLoad Time: 0.0018s | Model Forward Time: 5.5487s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0034 | max_forward: 0.2002 | max_loss: 0.0036 | max_backward: 0.3668 | max_optimizer_step: 0.2700 | max_tensorboard_logging: 3.9098 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0824 | Memory: rss_MB: 5443.1 | uss_MB: 4393.2 | child_rss_MB: 10497.0 | allocated_MB: 618.5 | reserved_MB: 28584.0 | inactive_split_MB: 893.5 +rank5 [2026-10-27 17:11:07,917] INFO : Step 15000/100000 rank5 | Train Loss: 13.2483 | Train Loss Unweighted: 4.5172 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.6965 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3441 | Regression_eta: 0.0751 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7702 | physical_nopu: 4.6706 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3817 | Classification_binary: 3.9281 | Regression_cos_phi: 2.3987 | Regression_energy: 1.7275 | Regression_eta: 0.9751 | Regression_pt: 2.0344 | Regression_sin_phi: 2.1687 | LR: 3.80e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.3447s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0879 | max_loss: 0.0036 | max_backward: 0.0859 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 2.2540 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0190 | Memory: rss_MB: 5437.1 | uss_MB: 4385.0 | child_rss_MB: 10498.2 | allocated_MB: 605.9 | reserved_MB: 28584.0 | inactive_split_MB: 314.1 +rank5 [2026-10-27 17:14:55,829] INFO : Step 16000/100000 rank5 | Train Loss: 14.0060 | Train Loss Unweighted: 4.5724 | Train Loss Components: Classification: 0.0150 | Classification_binary: 3.5827 | Regression_cos_phi: 0.0469 | Regression_energy: 0.4276 | Regression_eta: 0.0760 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0517 | Group Train Loss Unweighted (window mean): physical_pu: 3.7654 | physical_nopu: 4.6500 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4271 | Classification_binary: 3.5354 | Regression_cos_phi: 1.4776 | Regression_energy: 2.0387 | Regression_eta: 1.2626 | Regression_pt: 2.2566 | Regression_sin_phi: 2.2010 | LR: 3.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.4622s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0984 | max_forward: 0.1749 | max_loss: 0.0039 | max_backward: 0.0976 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 2.1733 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5435.1 | uss_MB: 4384.3 | child_rss_MB: 10495.1 | allocated_MB: 607.6 | reserved_MB: 28978.0 | inactive_split_MB: 330.4 +rank5 [2026-10-27 17:18:33,642] INFO : Step 17000/100000 rank5 | Train Loss: 14.2557 | Train Loss Unweighted: 4.6663 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6292 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4540 | Regression_eta: 0.0715 | Regression_pt: 0.3998 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.7576 | physical_nopu: 4.6430 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3849 | Classification_binary: 3.6468 | Regression_cos_phi: 1.8865 | Regression_energy: 2.5362 | Regression_eta: 1.6973 | Regression_pt: 2.9217 | Regression_sin_phi: 1.7180 | LR: 3.75e-04 | DataLoad Time: 0.5709s | Model Forward Time: 3.4022s | Phase Times: max_pre_step_barrier: 0.5705 | max_batch_to_device: 0.0043 | max_forward: 0.0887 | max_loss: 0.0037 | max_backward: 0.1154 | max_optimizer_step: 0.2542 | max_tensorboard_logging: 2.1801 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5421.0 | uss_MB: 4384.2 | child_rss_MB: 10406.9 | allocated_MB: 609.7 | reserved_MB: 29500.0 | inactive_split_MB: 302.1 +rank5 [2026-10-27 17:22:18,273] INFO : Step 18000/100000 rank5 | Train Loss: 13.2896 | Train Loss Unweighted: 4.5062 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6504 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3607 | Regression_eta: 0.0732 | Regression_pt: 0.3062 | Regression_sin_phi: 0.0530 | Group Train Loss Unweighted (window mean): physical_pu: 3.7500 | physical_nopu: 4.6286 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4425 | Classification_binary: 3.7039 | Regression_cos_phi: 1.7740 | Regression_energy: 1.5325 | Regression_eta: 0.9842 | Regression_pt: 1.7370 | Regression_sin_phi: 1.8241 | LR: 3.72e-04 | DataLoad Time: 0.0020s | Model Forward Time: 10.0051s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0832 | max_loss: 0.0037 | max_backward: 0.0999 | max_optimizer_step: 0.1725 | max_tensorboard_logging: 8.8254 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 5416.7 | uss_MB: 4384.0 | child_rss_MB: 10363.7 | allocated_MB: 608.8 | reserved_MB: 29500.0 | inactive_split_MB: 196.1 +rank5 [2026-10-27 17:25:51,406] INFO : Step 19000/100000 rank5 | Train Loss: 15.2449 | Train Loss Unweighted: 4.8480 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5669 | Regression_cos_phi: 0.0399 | Regression_energy: 0.5792 | Regression_eta: 0.0602 | Regression_pt: 0.5415 | Regression_sin_phi: 0.0443 | Group Train Loss Unweighted (window mean): physical_pu: 3.7399 | physical_nopu: 4.6217 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5436 | Classification_binary: 3.5078 | Regression_cos_phi: 1.4910 | Regression_energy: 4.3087 | Regression_eta: 0.8447 | Regression_pt: 5.6470 | Regression_sin_phi: 1.4119 | LR: 3.69e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.1647s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0008 | max_forward: 0.0818 | max_loss: 0.0762 | max_backward: 0.1759 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 1.9822 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5410.7 | uss_MB: 4384.1 | child_rss_MB: 10330.8 | allocated_MB: 607.1 | reserved_MB: 29500.0 | inactive_split_MB: 236.2 +rank5 [2026-10-27 17:29:45,026] INFO : Step 20000/100000 rank5 | Train Loss: 13.7275 | Train Loss Unweighted: 4.6495 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.7120 | Regression_cos_phi: 0.0510 | Regression_energy: 0.4061 | Regression_eta: 0.0646 | Regression_pt: 0.3531 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.7387 | physical_nopu: 4.6038 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3913 | Classification_binary: 3.5145 | Regression_cos_phi: 1.9612 | Regression_energy: 2.2075 | Regression_eta: 1.2321 | Regression_pt: 2.5737 | Regression_sin_phi: 2.0192 | LR: 3.66e-04 | DataLoad Time: 0.0027s | Model Forward Time: 5.4933s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0007 | max_forward: 0.0851 | max_loss: 0.0039 | max_backward: 0.2016 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 4.2823 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5411.1 | uss_MB: 4383.8 | child_rss_MB: 10307.1 | allocated_MB: 609.5 | reserved_MB: 29500.0 | inactive_split_MB: 252.8 +rank5 [2026-10-27 17:29:45,970] INFO : Running validation on rank5 +rank5 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 17:31:42,604] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 17:31:43,254] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 17:32:09,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 17:32:10,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 17:32:10,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 17:32:11,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 17:32:11,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 17:32:12,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 17:32:12,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 17:32:13,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 17:32:23,483] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 17:32:23,636] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 17:32:24,023] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 17:32:39,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 17:32:40,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 17:32:40,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 17:32:41,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 17:32:41,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 17:32:42,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 17:32:42,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 17:32:43,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 17:32:53,252] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 17:32:53,432] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 17:32:53,822] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 17:33:09,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 17:33:09,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 17:33:10,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 17:33:10,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 17:33:11,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 17:33:11,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 17:33:12,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 17:33:12,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 17:33:22,645] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 17:37:33,418] INFO : Step 21000/100000 rank5 | Train Loss: 13.1647 | Train Loss Unweighted: 4.3967 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.5106 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3742 | Regression_eta: 0.0796 | Regression_pt: 0.3229 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.7328 | physical_nopu: 4.5933 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5677 | Classification_binary: 3.4291 | Regression_cos_phi: 1.7871 | Regression_energy: 2.0830 | Regression_eta: 0.9557 | Regression_pt: 2.3733 | Regression_sin_phi: 1.8815 | LR: 3.62e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.6672s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0483 | max_forward: 0.1678 | max_loss: 0.0037 | max_backward: 0.1430 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 2.3569 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5516.5 | uss_MB: 4216.2 | child_rss_MB: 10241.6 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 238.6 +rank5 [2026-10-27 17:41:10,019] INFO : Step 22000/100000 rank5 | Train Loss: 15.3802 | Train Loss Unweighted: 4.7914 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.5311 | Regression_cos_phi: 0.0475 | Regression_energy: 0.5593 | Regression_eta: 0.0730 | Regression_pt: 0.5219 | Regression_sin_phi: 0.0434 | Group Train Loss Unweighted (window mean): physical_pu: 3.7290 | physical_nopu: 4.5858 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3554 | Classification_binary: 3.6664 | Regression_cos_phi: 1.7368 | Regression_energy: 1.7935 | Regression_eta: 0.8679 | Regression_pt: 2.0226 | Regression_sin_phi: 1.5599 | LR: 3.59e-04 | DataLoad Time: 0.0007s | Model Forward Time: 3.1753s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0905 | max_loss: 0.0038 | max_backward: 0.1361 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 2.0169 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5517.5 | uss_MB: 4498.8 | child_rss_MB: 10247.2 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 321.0 +rank5 [2026-10-27 17:44:47,651] INFO : Step 23000/100000 rank5 | Train Loss: 12.9848 | Train Loss Unweighted: 4.5226 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.7377 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3326 | Regression_eta: 0.0739 | Regression_pt: 0.2617 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.7288 | physical_nopu: 4.5774 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3233 | Classification_binary: 3.7448 | Regression_cos_phi: 1.8942 | Regression_energy: 2.0713 | Regression_eta: 1.4137 | Regression_pt: 2.0669 | Regression_sin_phi: 2.2745 | LR: 3.55e-04 | DataLoad Time: 0.0068s | Model Forward Time: 3.1591s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0994 | max_loss: 0.0039 | max_backward: 0.1244 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.9838 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 5517.5 | uss_MB: 4498.0 | child_rss_MB: 10251.2 | allocated_MB: 608.2 | reserved_MB: 29422.0 | inactive_split_MB: 218.5 +rank5 [2026-10-27 17:48:44,022] INFO : Step 24000/100000 rank5 | Train Loss: 16.3896 | Train Loss Unweighted: 4.8949 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4510 | Regression_cos_phi: 0.0465 | Regression_energy: 0.6613 | Regression_eta: 0.0617 | Regression_pt: 0.6132 | Regression_sin_phi: 0.0453 | Group Train Loss Unweighted (window mean): physical_pu: 3.7170 | physical_nopu: 4.5767 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6032 | Classification_binary: 3.1292 | Regression_cos_phi: 0.9110 | Regression_energy: 5.3919 | Regression_eta: 0.5877 | Regression_pt: 7.0914 | Regression_sin_phi: 0.9434 | LR: 3.51e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.1028s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0398 | max_forward: 0.1314 | max_loss: 0.0037 | max_backward: 0.0994 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 1.8290 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5518.7 | uss_MB: 4498.4 | child_rss_MB: 10238.8 | allocated_MB: 606.3 | reserved_MB: 29422.0 | inactive_split_MB: 288.6 +rank5 [2026-10-27 17:52:29,098] INFO : Step 25000/100000 rank5 | Train Loss: 13.0062 | Train Loss Unweighted: 4.4048 | Train Loss Components: Classification: 0.0114 | Classification_binary: 3.6384 | Regression_cos_phi: 0.0552 | Regression_energy: 0.3153 | Regression_eta: 0.0680 | Regression_pt: 0.2574 | Regression_sin_phi: 0.0590 | Group Train Loss Unweighted (window mean): physical_pu: 3.7138 | physical_nopu: 4.5644 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3036 | Classification_binary: 3.7313 | Regression_cos_phi: 2.4002 | Regression_energy: 1.6725 | Regression_eta: 0.8243 | Regression_pt: 2.0675 | Regression_sin_phi: 2.4923 | LR: 3.47e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.5990s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0007 | max_forward: 0.1394 | max_loss: 0.0036 | max_backward: 0.2004 | max_optimizer_step: 0.1603 | max_tensorboard_logging: 1.3791 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5517.7 | uss_MB: 4498.3 | child_rss_MB: 10255.6 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 213.9 +rank5 [2026-10-27 17:56:10,619] INFO : Step 26000/100000 rank5 | Train Loss: 13.5469 | Train Loss Unweighted: 4.4523 | Train Loss Components: Classification: 0.0185 | Classification_binary: 3.5246 | Regression_cos_phi: 0.0498 | Regression_energy: 0.3935 | Regression_eta: 0.0732 | Regression_pt: 0.3461 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5624 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5618 | Classification_binary: 3.6875 | Regression_cos_phi: 1.8819 | Regression_energy: 2.1726 | Regression_eta: 1.3187 | Regression_pt: 2.3394 | Regression_sin_phi: 2.0435 | LR: 3.43e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.1213s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0838 | max_forward: 0.1679 | max_loss: 0.0039 | max_backward: 0.0991 | max_optimizer_step: 0.2674 | max_tensorboard_logging: 1.6325 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5515.7 | uss_MB: 4498.4 | child_rss_MB: 10248.9 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 201.9 +rank5 [2026-10-27 17:59:48,627] INFO : Step 27000/100000 rank5 | Train Loss: 17.1780 | Train Loss Unweighted: 5.0531 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.5146 | Regression_cos_phi: 0.0448 | Regression_energy: 0.6955 | Regression_eta: 0.0778 | Regression_pt: 0.6583 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5579 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4022 | Classification_binary: 3.5941 | Regression_cos_phi: 1.6948 | Regression_energy: 4.1745 | Regression_eta: 1.4140 | Regression_pt: 5.2924 | Regression_sin_phi: 1.9210 | LR: 3.39e-04 | DataLoad Time: 0.0011s | Model Forward Time: 2.4484s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1019 | max_loss: 0.0489 | max_backward: 0.1596 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5507.7 | uss_MB: 4499.5 | child_rss_MB: 10244.8 | allocated_MB: 611.7 | reserved_MB: 30248.0 | inactive_split_MB: 336.3 +rank5 [2026-10-27 18:03:26,754] INFO : Step 28000/100000 rank5 | Train Loss: 12.6721 | Train Loss Unweighted: 4.3436 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5550 | Regression_cos_phi: 0.0488 | Regression_energy: 0.3264 | Regression_eta: 0.0754 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7057 | physical_nopu: 4.5482 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3136 | Classification_binary: 3.5736 | Regression_cos_phi: 1.5365 | Regression_energy: 1.8238 | Regression_eta: 1.0964 | Regression_pt: 2.0624 | Regression_sin_phi: 1.9161 | LR: 3.35e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.0166s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0898 | max_loss: 0.0057 | max_backward: 0.1164 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.8021 | max_tensorboard_flush: 0.0040 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 5489.8 | uss_MB: 4496.9 | child_rss_MB: 10097.2 | allocated_MB: 606.6 | reserved_MB: 30248.0 | inactive_split_MB: 278.7 +rank5 [2026-10-27 18:07:25,425] INFO : Step 29000/100000 rank5 | Train Loss: 14.8403 | Train Loss Unweighted: 4.5753 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.4458 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4901 | Regression_eta: 0.0767 | Regression_pt: 0.4459 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.7068 | physical_nopu: 4.5439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5469 | Classification_binary: 3.5570 | Regression_cos_phi: 2.2225 | Regression_energy: 2.3035 | Regression_eta: 1.1684 | Regression_pt: 2.6411 | Regression_sin_phi: 2.5616 | LR: 3.30e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.3381s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1464 | max_loss: 0.0037 | max_backward: 0.0935 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 1.1535 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5489.8 | uss_MB: 4498.4 | child_rss_MB: 10094.9 | allocated_MB: 607.4 | reserved_MB: 30248.0 | inactive_split_MB: 313.3 +rank5 [2026-10-27 18:11:12,494] INFO : Step 30000/100000 rank5 | Train Loss: 13.2916 | Train Loss Unweighted: 4.4954 | Train Loss Components: Classification: 0.0126 | Classification_binary: 3.5989 | Regression_cos_phi: 0.0482 | Regression_energy: 0.3865 | Regression_eta: 0.0707 | Regression_pt: 0.3291 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7005 | physical_nopu: 4.5378 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3587 | Classification_binary: 3.5161 | Regression_cos_phi: 1.6299 | Regression_energy: 1.6137 | Regression_eta: 1.0262 | Regression_pt: 1.9360 | Regression_sin_phi: 1.9822 | LR: 3.26e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.7232s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0805 | max_loss: 0.0038 | max_backward: 0.0926 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 1.5279 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 5479.8 | uss_MB: 4498.3 | child_rss_MB: 10016.6 | allocated_MB: 608.2 | reserved_MB: 30760.0 | inactive_split_MB: 232.9 +rank5 [2026-10-27 18:11:13,431] INFO : Running validation on rank5 +rank5 [2026-10-27 18:11:13,432] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 18:11:13,432] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 18:11:13,432] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 18:11:13,432] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 18:13:07,587] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 18:13:08,047] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 18:13:34,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 18:13:35,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 18:13:35,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 18:13:36,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 18:13:36,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 18:13:37,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 18:13:37,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 18:13:38,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 18:13:48,102] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 18:13:48,509] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 18:13:49,356] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 18:14:05,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 18:14:05,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 18:14:06,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 18:14:06,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 18:14:07,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 18:14:07,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 18:14:08,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 18:14:08,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 18:14:18,739] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 18:14:18,746] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 18:14:19,219] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 18:14:34,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 18:14:35,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 18:14:35,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 18:14:36,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 18:14:36,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 18:14:37,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 18:14:37,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 18:14:38,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 18:14:48,055] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 18:19:00,303] INFO : Step 31000/100000 rank5 | Train Loss: 12.7750 | Train Loss Unweighted: 4.3889 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5406 | Regression_cos_phi: 0.0465 | Regression_energy: 0.3622 | Regression_eta: 0.0700 | Regression_pt: 0.3101 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6964 | physical_nopu: 4.5360 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4674 | Classification_binary: 3.6277 | Regression_cos_phi: 1.6363 | Regression_energy: 1.6917 | Regression_eta: 0.9961 | Regression_pt: 2.0462 | Regression_sin_phi: 2.0556 | LR: 3.21e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.3235s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.2346 | max_loss: 0.0037 | max_backward: 0.1255 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 1.8384 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1395 | Memory: rss_MB: 6773.5 | uss_MB: 5791.8 | child_rss_MB: 10011.8 | allocated_MB: 609.3 | reserved_MB: 29142.0 | inactive_split_MB: 278.5 +rank5 [2026-10-27 18:22:44,788] INFO : Step 32000/100000 rank5 | Train Loss: 14.3617 | Train Loss Unweighted: 4.6225 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4934 | Regression_cos_phi: 0.0448 | Regression_energy: 0.5043 | Regression_eta: 0.0718 | Regression_pt: 0.4507 | Regression_sin_phi: 0.0414 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5314 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3818 | Classification_binary: 3.5608 | Regression_cos_phi: 1.5001 | Regression_energy: 2.2359 | Regression_eta: 1.0789 | Regression_pt: 2.3800 | Regression_sin_phi: 1.6372 | LR: 3.16e-04 | DataLoad Time: 0.0112s | Model Forward Time: 2.5616s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1373 | max_loss: 0.0041 | max_backward: 0.2040 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 1.2065 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5556.1 | uss_MB: 4574.8 | child_rss_MB: 10008.9 | allocated_MB: 606.1 | reserved_MB: 29142.0 | inactive_split_MB: 511.2 +rank5 [2026-10-27 18:26:33,026] INFO : Step 33000/100000 rank5 | Train Loss: 12.0000 | Train Loss Unweighted: 3.7115 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9674 | Regression_cos_phi: 0.0474 | Regression_energy: 0.2863 | Regression_eta: 0.1056 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4742 | Classification_binary: 2.9429 | Regression_cos_phi: 1.7528 | Regression_energy: 1.6180 | Regression_eta: 1.7009 | Regression_pt: 1.8296 | Regression_sin_phi: 1.9095 | LR: 3.12e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.5987s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0030 | max_forward: 0.1619 | max_loss: 0.0119 | max_backward: 0.2845 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.1927 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5557.1 | uss_MB: 4556.3 | child_rss_MB: 10013.2 | allocated_MB: 623.7 | reserved_MB: 29572.0 | inactive_split_MB: 1441.5 +rank5 [2026-10-27 18:30:15,971] INFO : Step 34000/100000 rank5 | Train Loss: 15.0627 | Train Loss Unweighted: 4.6899 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4670 | Regression_cos_phi: 0.0454 | Regression_energy: 0.5438 | Regression_eta: 0.0626 | Regression_pt: 0.5082 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6889 | physical_nopu: 4.5215 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4562 | Classification_binary: 3.5238 | Regression_cos_phi: 1.8827 | Regression_energy: 2.9441 | Regression_eta: 0.9191 | Regression_pt: 3.6126 | Regression_sin_phi: 2.1098 | LR: 3.07e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.3754s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0007 | max_forward: 0.1346 | max_loss: 0.0037 | max_backward: 0.1938 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1506 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5552.6 | uss_MB: 4573.6 | child_rss_MB: 9984.9 | allocated_MB: 609.6 | reserved_MB: 29572.0 | inactive_split_MB: 230.4 +rank5 [2026-10-27 18:33:55,272] INFO : Step 35000/100000 rank5 | Train Loss: 12.2019 | Train Loss Unweighted: 4.3540 | Train Loss Components: Classification: 0.0089 | Classification_binary: 3.6426 | Regression_cos_phi: 0.0532 | Regression_energy: 0.2965 | Regression_eta: 0.0664 | Regression_pt: 0.2370 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6854 | physical_nopu: 4.5184 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2889 | Classification_binary: 3.7454 | Regression_cos_phi: 1.8906 | Regression_energy: 1.5487 | Regression_eta: 1.1277 | Regression_pt: 1.5190 | Regression_sin_phi: 2.7514 | LR: 3.02e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.6472s | Phase Times: max_pre_step_barrier: 0.0511 | max_batch_to_device: 0.0857 | max_forward: 0.1832 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 1.2656 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 5548.6 | uss_MB: 4574.4 | child_rss_MB: 9918.7 | allocated_MB: 608.2 | reserved_MB: 29572.0 | inactive_split_MB: 245.8 +rank5 [2026-10-27 18:37:33,850] INFO : Step 36000/100000 rank5 | Train Loss: 12.7130 | Train Loss Unweighted: 4.3877 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.5479 | Regression_cos_phi: 0.0433 | Regression_energy: 0.3521 | Regression_eta: 0.0636 | Regression_pt: 0.3167 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.6860 | physical_nopu: 4.5137 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5612 | Classification_binary: 3.5255 | Regression_cos_phi: 1.5506 | Regression_energy: 1.8475 | Regression_eta: 0.9455 | Regression_pt: 2.1803 | Regression_sin_phi: 2.0036 | LR: 2.97e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.5818s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0007 | max_forward: 0.1685 | max_loss: 0.0137 | max_backward: 0.0851 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.2906 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 5532.6 | uss_MB: 4487.8 | child_rss_MB: 9818.0 | allocated_MB: 609.0 | reserved_MB: 29572.0 | inactive_split_MB: 268.4 +rank5 [2026-10-27 18:41:12,499] INFO : Step 37000/100000 rank5 | Train Loss: 16.1273 | Train Loss Unweighted: 4.7522 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4049 | Regression_cos_phi: 0.0495 | Regression_energy: 0.5975 | Regression_eta: 0.0881 | Regression_pt: 0.5494 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6829 | physical_nopu: 4.5107 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4553 | Classification_binary: 3.4068 | Regression_cos_phi: 1.5739 | Regression_energy: 2.9471 | Regression_eta: 0.8374 | Regression_pt: 3.8906 | Regression_sin_phi: 1.5133 | LR: 2.91e-04 | DataLoad Time: 0.0027s | Model Forward Time: 2.3164s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0884 | max_loss: 0.0036 | max_backward: 0.0963 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 1.0635 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0656 | Memory: rss_MB: 5529.2 | uss_MB: 4574.6 | child_rss_MB: 9790.1 | allocated_MB: 605.6 | reserved_MB: 29572.0 | inactive_split_MB: 233.2 +rank5 [2026-10-27 18:44:44,282] INFO : Step 38000/100000 rank5 | Train Loss: 12.5142 | Train Loss Unweighted: 4.4218 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.6290 | Regression_cos_phi: 0.0475 | Regression_energy: 0.3376 | Regression_eta: 0.0660 | Regression_pt: 0.2827 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6875 | physical_nopu: 4.5136 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3434 | Classification_binary: 3.5357 | Regression_cos_phi: 2.1521 | Regression_energy: 2.6204 | Regression_eta: 0.8740 | Regression_pt: 3.0769 | Regression_sin_phi: 1.6654 | LR: 2.86e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.4322s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.0807 | max_loss: 0.0039 | max_backward: 0.1583 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.2386 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5519.2 | uss_MB: 4574.3 | child_rss_MB: 9713.9 | allocated_MB: 607.7 | reserved_MB: 29572.0 | inactive_split_MB: 203.1 +rank5 [2026-10-27 18:48:30,202] INFO : Step 39000/100000 rank5 | Train Loss: 13.9834 | Train Loss Unweighted: 4.4509 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4391 | Regression_cos_phi: 0.0477 | Regression_energy: 0.4363 | Regression_eta: 0.0792 | Regression_pt: 0.3839 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.6802 | physical_nopu: 4.5058 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5601 | Classification_binary: 3.6235 | Regression_cos_phi: 1.9825 | Regression_energy: 2.2552 | Regression_eta: 1.1823 | Regression_pt: 2.5622 | Regression_sin_phi: 1.9515 | LR: 2.81e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.8724s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.0950 | max_loss: 0.0039 | max_backward: 0.1102 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 2.6970 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5506.6 | uss_MB: 4574.6 | child_rss_MB: 9623.2 | allocated_MB: 611.4 | reserved_MB: 29682.0 | inactive_split_MB: 276.6 +rank5 [2026-10-27 18:52:07,824] INFO : Step 40000/100000 rank5 | Train Loss: 15.4467 | Train Loss Unweighted: 4.8241 | Train Loss Components: Classification: 0.0121 | Classification_binary: 3.5538 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5661 | Regression_eta: 0.0744 | Regression_pt: 0.5256 | Regression_sin_phi: 0.0462 | Group Train Loss Unweighted (window mean): physical_pu: 3.6754 | physical_nopu: 4.5045 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5222 | Classification_binary: 3.3759 | Regression_cos_phi: 1.5997 | Regression_energy: 6.5563 | Regression_eta: 0.8912 | Regression_pt: 8.4755 | Regression_sin_phi: 1.6805 | LR: 2.76e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4741s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.1251 | max_loss: 0.0037 | max_backward: 0.0913 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2799 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5491.6 | uss_MB: 4574.5 | child_rss_MB: 9510.6 | allocated_MB: 607.6 | reserved_MB: 29682.0 | inactive_split_MB: 251.4 +rank5 [2026-10-27 18:52:08,683] INFO : Running validation on rank5 +rank5 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 18:52:08,684] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 18:52:08,684] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 18:52:08,684] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 18:54:11,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 18:54:12,297] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 18:54:37,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 18:54:38,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 18:54:38,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 18:54:39,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 18:54:39,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 18:54:40,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 18:54:40,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 18:54:41,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 18:54:51,227] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 18:54:51,320] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 18:54:51,717] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 18:55:07,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 18:55:08,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 18:55:08,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 18:55:09,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 18:55:09,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 18:55:10,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 18:55:10,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 18:55:11,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 18:55:21,155] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 18:55:21,167] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 18:55:21,564] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 18:55:36,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 18:55:37,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 18:55:38,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 18:55:38,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 18:55:39,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 18:55:39,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 18:55:40,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 18:55:40,515] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 18:55:50,524] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 19:00:08,492] INFO : Step 41000/100000 rank5 | Train Loss: 13.2101 | Train Loss Unweighted: 4.4466 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5620 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3777 | Regression_eta: 0.0720 | Regression_pt: 0.3238 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6774 | physical_nopu: 4.4925 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4580 | Classification_binary: 3.5004 | Regression_cos_phi: 2.4530 | Regression_energy: 1.8846 | Regression_eta: 1.2328 | Regression_pt: 2.1087 | Regression_sin_phi: 2.0131 | LR: 2.70e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.5585s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0904 | max_forward: 0.2227 | max_loss: 0.0038 | max_backward: 0.1239 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 2.1580 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6748.4 | uss_MB: 5831.9 | child_rss_MB: 9514.0 | allocated_MB: 608.3 | reserved_MB: 27230.0 | inactive_split_MB: 387.2 +rank5 [2026-10-27 19:03:45,853] INFO : Step 42000/100000 rank5 | Train Loss: 13.7061 | Train Loss Unweighted: 4.4760 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4108 | Regression_cos_phi: 0.0459 | Regression_energy: 0.4702 | Regression_eta: 0.0614 | Regression_pt: 0.4341 | Regression_sin_phi: 0.0374 | Group Train Loss Unweighted (window mean): physical_pu: 3.6711 | physical_nopu: 4.4967 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6348 | Classification_binary: 3.3724 | Regression_cos_phi: 2.0542 | Regression_energy: 3.7724 | Regression_eta: 1.1603 | Regression_pt: 4.9304 | Regression_sin_phi: 1.3005 | LR: 2.65e-04 | DataLoad Time: 0.0066s | Model Forward Time: 2.9862s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0008 | max_forward: 0.1920 | max_loss: 0.0038 | max_backward: 0.1427 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 1.6213 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6739.4 | uss_MB: 5628.8 | child_rss_MB: 9436.4 | allocated_MB: 609.4 | reserved_MB: 27632.0 | inactive_split_MB: 509.5 +rank5 [2026-10-27 19:07:23,506] INFO : Step 43000/100000 rank5 | Train Loss: 11.9489 | Train Loss Unweighted: 3.6938 | Train Loss Components: Classification: 0.0162 | Classification_binary: 2.9682 | Regression_cos_phi: 0.0472 | Regression_energy: 0.2764 | Regression_eta: 0.1089 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6702 | physical_nopu: 4.4944 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5200 | Classification_binary: 2.9984 | Regression_cos_phi: 1.7077 | Regression_energy: 1.4860 | Regression_eta: 1.7211 | Regression_pt: 1.6081 | Regression_sin_phi: 1.8196 | LR: 2.59e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.8331s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0032 | max_forward: 0.2195 | max_loss: 0.0114 | max_backward: 0.4513 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 2.0889 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0865 | Memory: rss_MB: 5560.5 | uss_MB: 3524.0 | child_rss_MB: 9348.2 | allocated_MB: 619.9 | reserved_MB: 27918.0 | inactive_split_MB: 1305.5 +rank5 [2026-10-27 19:11:14,852] INFO : Step 44000/100000 rank5 | Train Loss: 15.8790 | Train Loss Unweighted: 4.7893 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4610 | Regression_cos_phi: 0.0438 | Regression_energy: 0.5944 | Regression_eta: 0.0747 | Regression_pt: 0.5488 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6681 | physical_nopu: 4.4856 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.7232 | Classification_binary: 3.4709 | Regression_cos_phi: 1.8077 | Regression_energy: 3.3448 | Regression_eta: 1.4326 | Regression_pt: 3.3671 | Regression_sin_phi: 3.3380 | LR: 2.54e-04 | DataLoad Time: 0.0027s | Model Forward Time: 3.1647s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0980 | max_loss: 0.0531 | max_backward: 0.1371 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.9403 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5553.5 | uss_MB: 4666.8 | child_rss_MB: 9287.7 | allocated_MB: 606.4 | reserved_MB: 31154.0 | inactive_split_MB: 295.0 +rank5 [2026-10-27 19:15:01,382] INFO : Step 45000/100000 rank5 | Train Loss: 13.8786 | Train Loss Unweighted: 4.5241 | Train Loss Components: Classification: 0.0100 | Classification_binary: 3.5822 | Regression_cos_phi: 0.0536 | Regression_energy: 0.4068 | Regression_eta: 0.0847 | Regression_pt: 0.3344 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 3.6667 | physical_nopu: 4.4836 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3628 | Classification_binary: 3.4537 | Regression_cos_phi: 2.3256 | Regression_energy: 3.1549 | Regression_eta: 3.1184 | Regression_pt: 3.2745 | Regression_sin_phi: 3.1514 | LR: 2.48e-04 | DataLoad Time: 0.0016s | Model Forward Time: 4.6039s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0009 | max_forward: 0.0951 | max_loss: 0.0815 | max_backward: 0.1837 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 3.3466 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 5551.4 | uss_MB: 4666.0 | child_rss_MB: 9284.1 | allocated_MB: 608.8 | reserved_MB: 31154.0 | inactive_split_MB: 289.8 +rank5 [2026-10-27 19:18:42,715] INFO : Step 46000/100000 rank5 | Train Loss: 13.1587 | Train Loss Unweighted: 4.3779 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.4817 | Regression_cos_phi: 0.0459 | Regression_energy: 0.3805 | Regression_eta: 0.0668 | Regression_pt: 0.3375 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.6620 | physical_nopu: 4.4831 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4669 | Classification_binary: 3.5119 | Regression_cos_phi: 1.9728 | Regression_energy: 2.0385 | Regression_eta: 0.9518 | Regression_pt: 2.4739 | Regression_sin_phi: 2.4578 | LR: 2.43e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.3332s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0924 | max_forward: 0.1910 | max_loss: 0.0039 | max_backward: 0.1028 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 2.0151 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5545.4 | uss_MB: 4666.2 | child_rss_MB: 9248.1 | allocated_MB: 609.2 | reserved_MB: 31154.0 | inactive_split_MB: 277.4 +rank5 [2026-10-27 19:22:31,522] INFO : Step 47000/100000 rank5 | Train Loss: 15.2118 | Train Loss Unweighted: 4.6831 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3624 | Regression_cos_phi: 0.0409 | Regression_energy: 0.5987 | Regression_eta: 0.0659 | Regression_pt: 0.5598 | Regression_sin_phi: 0.0405 | Group Train Loss Unweighted (window mean): physical_pu: 3.6609 | physical_nopu: 4.4795 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3435 | Classification_binary: 3.3038 | Regression_cos_phi: 1.7849 | Regression_energy: 3.2187 | Regression_eta: 1.1452 | Regression_pt: 3.7278 | Regression_sin_phi: 2.0780 | LR: 2.37e-04 | DataLoad Time: 0.0014s | Model Forward Time: 3.2748s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0777 | max_forward: 0.1864 | max_loss: 0.0077 | max_backward: 0.1105 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 1.8392 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5533.2 | uss_MB: 4665.8 | child_rss_MB: 9129.7 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 305.0 +rank5 [2026-10-27 19:26:21,057] INFO : Step 48000/100000 rank5 | Train Loss: 12.0455 | Train Loss Unweighted: 4.3014 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.6150 | Regression_cos_phi: 0.0526 | Regression_energy: 0.2805 | Regression_eta: 0.0652 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6617 | physical_nopu: 4.4726 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2734 | Classification_binary: 3.5735 | Regression_cos_phi: 1.2651 | Regression_energy: 1.4275 | Regression_eta: 0.7566 | Regression_pt: 1.5781 | Regression_sin_phi: 1.8231 | LR: 2.31e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.3649s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0531 | max_forward: 0.1607 | max_loss: 0.0044 | max_backward: 0.0997 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 2.0345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5534.2 | uss_MB: 4664.3 | child_rss_MB: 9129.2 | allocated_MB: 606.8 | reserved_MB: 31154.0 | inactive_split_MB: 262.3 +rank5 [2026-10-27 19:30:06,213] INFO : Step 49000/100000 rank5 | Train Loss: 13.3973 | Train Loss Unweighted: 4.4035 | Train Loss Components: Classification: 0.0168 | Classification_binary: 3.5033 | Regression_cos_phi: 0.0506 | Regression_energy: 0.3797 | Regression_eta: 0.0752 | Regression_pt: 0.3299 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6566 | physical_nopu: 4.4735 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5278 | Classification_binary: 3.5069 | Regression_cos_phi: 1.7660 | Regression_energy: 1.9152 | Regression_eta: 0.7274 | Regression_pt: 2.2779 | Regression_sin_phi: 1.8011 | LR: 2.26e-04 | DataLoad Time: 0.0020s | Model Forward Time: 4.5486s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0220 | max_forward: 0.1075 | max_loss: 0.0037 | max_backward: 0.1020 | max_optimizer_step: 0.2722 | max_tensorboard_logging: 3.2606 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5532.9 | uss_MB: 4389.0 | child_rss_MB: 9133.5 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 312.3 +rank5 [2026-10-27 19:33:47,825] INFO : Step 50000/100000 rank5 | Train Loss: 14.2027 | Train Loss Unweighted: 4.5299 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4272 | Regression_cos_phi: 0.0432 | Regression_energy: 0.4877 | Regression_eta: 0.0670 | Regression_pt: 0.4429 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4710 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3998 | Classification_binary: 3.4507 | Regression_cos_phi: 1.5178 | Regression_energy: 3.1729 | Regression_eta: 0.9218 | Regression_pt: 3.9173 | Regression_sin_phi: 1.5331 | LR: 2.20e-04 | DataLoad Time: 0.0026s | Model Forward Time: 3.0497s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0964 | max_loss: 0.0590 | max_backward: 0.1520 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.8031 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5519.9 | uss_MB: 4665.8 | child_rss_MB: 9015.1 | allocated_MB: 607.2 | reserved_MB: 31154.0 | inactive_split_MB: 255.6 +rank5 [2026-10-27 19:33:48,729] INFO : Running validation on rank5 +rank5 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 19:35:46,023] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 19:35:46,495] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 19:36:12,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 19:36:12,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 19:36:13,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 19:36:13,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 19:36:14,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 19:36:14,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 19:36:15,263] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 19:36:15,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 19:36:25,767] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 19:36:25,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 19:36:26,396] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 19:36:41,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 19:36:42,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 19:36:42,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 19:36:43,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 19:36:43,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 19:36:44,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 19:36:44,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 19:36:45,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 19:36:55,359] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 19:36:55,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 19:36:55,960] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 19:37:10,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 19:37:11,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 19:37:11,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 19:37:12,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 19:37:12,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 19:37:13,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 19:37:13,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 19:37:14,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 19:37:24,229] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 19:41:35,792] INFO : Step 51000/100000 rank5 | Train Loss: 13.0997 | Train Loss Unweighted: 4.3733 | Train Loss Components: Classification: 0.0129 | Classification_binary: 3.5282 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3598 | Regression_eta: 0.0785 | Regression_pt: 0.2920 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6545 | physical_nopu: 4.4631 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4173 | Classification_binary: 3.5823 | Regression_cos_phi: 1.8166 | Regression_energy: 1.4245 | Regression_eta: 0.7992 | Regression_pt: 1.6309 | Regression_sin_phi: 1.7269 | LR: 2.14e-04 | DataLoad Time: 0.0103s | Model Forward Time: 3.1051s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1313 | max_loss: 0.0271 | max_backward: 0.1689 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.7590 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6858.3 | uss_MB: 6017.9 | child_rss_MB: 8958.7 | allocated_MB: 607.6 | reserved_MB: 29536.0 | inactive_split_MB: 388.7 +rank5 [2026-10-27 19:45:15,993] INFO : Step 52000/100000 rank5 | Train Loss: 11.7496 | Train Loss Unweighted: 3.6574 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9475 | Regression_cos_phi: 0.0469 | Regression_energy: 0.2717 | Regression_eta: 0.1056 | Regression_pt: 0.2231 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6567 | physical_nopu: 4.4685 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4647 | Classification_binary: 2.9518 | Regression_cos_phi: 1.7566 | Regression_energy: 1.5135 | Regression_eta: 1.6234 | Regression_pt: 1.6272 | Regression_sin_phi: 1.9142 | LR: 2.09e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.6142s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.1059 | max_forward: 0.2956 | max_loss: 0.0038 | max_backward: 0.4187 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 1.8177 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6858.3 | uss_MB: 6026.1 | child_rss_MB: 8942.2 | allocated_MB: 619.9 | reserved_MB: 29990.0 | inactive_split_MB: 1078.6 +rank5 [2026-10-27 19:48:51,528] INFO : Step 53000/100000 rank5 | Train Loss: 12.5294 | Train Loss Unweighted: 4.3083 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5302 | Regression_cos_phi: 0.0494 | Regression_energy: 0.3242 | Regression_eta: 0.0722 | Regression_pt: 0.2705 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6530 | physical_nopu: 4.4629 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4267 | Classification_binary: 3.5145 | Regression_cos_phi: 1.9505 | Regression_energy: 1.4873 | Regression_eta: 1.1074 | Regression_pt: 1.5069 | Regression_sin_phi: 2.0075 | LR: 2.03e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.5890s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.1740 | max_loss: 0.0036 | max_backward: 0.1135 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 1.3124 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 5606.5 | uss_MB: 4766.5 | child_rss_MB: 8943.1 | allocated_MB: 608.5 | reserved_MB: 29990.0 | inactive_split_MB: 246.4 +rank5 [2026-10-27 19:52:28,508] INFO : Step 54000/100000 rank5 | Train Loss: 13.6142 | Train Loss Unweighted: 4.3355 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.3742 | Regression_cos_phi: 0.0499 | Regression_energy: 0.4083 | Regression_eta: 0.0736 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4652 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4649 | Classification_binary: 3.4679 | Regression_cos_phi: 1.5699 | Regression_energy: 2.3925 | Regression_eta: 0.8567 | Regression_pt: 3.0572 | Regression_sin_phi: 1.8659 | LR: 1.97e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.4444s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1029 | max_loss: 0.0156 | max_backward: 0.1263 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2459 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 5600.5 | uss_MB: 4766.6 | child_rss_MB: 8886.2 | allocated_MB: 608.0 | reserved_MB: 29990.0 | inactive_split_MB: 249.1 +rank5 [2026-10-27 19:56:02,481] INFO : Step 55000/100000 rank5 | Train Loss: 14.4463 | Train Loss Unweighted: 4.6356 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5250 | Regression_cos_phi: 0.0497 | Regression_energy: 0.4959 | Regression_eta: 0.0660 | Regression_pt: 0.4403 | Regression_sin_phi: 0.0455 | Group Train Loss Unweighted (window mean): physical_pu: 3.6517 | physical_nopu: 4.4554 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6162 | Classification_binary: 3.4883 | Regression_cos_phi: 1.6034 | Regression_energy: 4.2701 | Regression_eta: 0.8290 | Regression_pt: 5.4440 | Regression_sin_phi: 1.3718 | LR: 1.92e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.9804s | Phase Times: max_pre_step_barrier: 0.0108 | max_batch_to_device: 0.0008 | max_forward: 0.0923 | max_loss: 0.0219 | max_backward: 0.1306 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 1.7796 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5599.8 | uss_MB: 4765.5 | child_rss_MB: 8883.4 | allocated_MB: 610.0 | reserved_MB: 29990.0 | inactive_split_MB: 274.0 +rank5 [2026-10-27 19:59:40,797] INFO : Step 56000/100000 rank5 | Train Loss: 12.4091 | Train Loss Unweighted: 4.2572 | Train Loss Components: Classification: 0.0135 | Classification_binary: 3.4523 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3421 | Regression_eta: 0.0664 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6493 | physical_nopu: 4.4547 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4772 | Classification_binary: 3.4537 | Regression_cos_phi: 1.6215 | Regression_energy: 1.3880 | Regression_eta: 0.7714 | Regression_pt: 1.5941 | Regression_sin_phi: 1.5918 | LR: 1.86e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.0682s | Phase Times: max_pre_step_barrier: 0.3290 | max_batch_to_device: 0.0902 | max_forward: 0.1771 | max_loss: 0.0038 | max_backward: 0.1058 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 1.4273 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5599.8 | uss_MB: 4766.9 | child_rss_MB: 8880.9 | allocated_MB: 608.1 | reserved_MB: 29990.0 | inactive_split_MB: 253.9 +rank5 [2026-10-27 20:03:29,496] INFO : Step 57000/100000 rank5 | Train Loss: 14.9087 | Train Loss Unweighted: 4.5957 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4371 | Regression_cos_phi: 0.0471 | Regression_energy: 0.5028 | Regression_eta: 0.0770 | Regression_pt: 0.4658 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6475 | physical_nopu: 4.4509 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5393 | Classification_binary: 3.3443 | Regression_cos_phi: 1.9267 | Regression_energy: 2.2890 | Regression_eta: 1.6124 | Regression_pt: 2.7586 | Regression_sin_phi: 2.1083 | LR: 1.81e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.4959s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1036 | max_loss: 0.0876 | max_backward: 0.2130 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 1.1951 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5597.8 | uss_MB: 4497.3 | child_rss_MB: 8866.5 | allocated_MB: 608.1 | reserved_MB: 30358.0 | inactive_split_MB: 283.9 +rank5 [2026-10-27 20:07:08,735] INFO : Step 58000/100000 rank5 | Train Loss: 12.4468 | Train Loss Unweighted: 4.2826 | Train Loss Components: Classification: 0.0105 | Classification_binary: 3.5299 | Regression_cos_phi: 0.0545 | Regression_energy: 0.3106 | Regression_eta: 0.0620 | Regression_pt: 0.2643 | Regression_sin_phi: 0.0509 | Group Train Loss Unweighted (window mean): physical_pu: 3.6440 | physical_nopu: 4.4492 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2278 | Classification_binary: 3.3855 | Regression_cos_phi: 2.2050 | Regression_energy: 1.4481 | Regression_eta: 1.0095 | Regression_pt: 1.7666 | Regression_sin_phi: 1.8116 | LR: 1.75e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.5425s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1868 | max_loss: 0.0037 | max_backward: 0.1111 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2860 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5587.6 | uss_MB: 3611.2 | child_rss_MB: 8803.1 | allocated_MB: 608.2 | reserved_MB: 30358.0 | inactive_split_MB: 337.0 +rank5 [2026-10-27 20:10:52,442] INFO : Step 59000/100000 rank5 | Train Loss: 13.1870 | Train Loss Unweighted: 4.3659 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4760 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3771 | Regression_eta: 0.0773 | Regression_pt: 0.3247 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4440 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5289 | Classification_binary: 3.5516 | Regression_cos_phi: 1.9071 | Regression_energy: 2.0821 | Regression_eta: 1.0725 | Regression_pt: 2.2696 | Regression_sin_phi: 1.9488 | LR: 1.70e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.4657s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1005 | max_loss: 0.0036 | max_backward: 0.1062 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 1.2390 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 5582.6 | uss_MB: 4766.7 | child_rss_MB: 8755.9 | allocated_MB: 610.1 | reserved_MB: 31234.0 | inactive_split_MB: 273.9 +rank5 [2026-10-27 20:14:34,818] INFO : Step 60000/100000 rank5 | Train Loss: 15.4000 | Train Loss Unweighted: 4.6905 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.3905 | Regression_cos_phi: 0.0456 | Regression_energy: 0.5805 | Regression_eta: 0.0703 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6406 | physical_nopu: 4.4523 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4384 | Classification_binary: 3.4963 | Regression_cos_phi: 1.9920 | Regression_energy: 2.5153 | Regression_eta: 0.9097 | Regression_pt: 2.9269 | Regression_sin_phi: 1.8564 | LR: 1.64e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.5441s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0757 | max_forward: 0.1624 | max_loss: 0.0040 | max_backward: 0.1315 | max_optimizer_step: 0.2155 | max_tensorboard_logging: 1.2531 | max_tensorboard_flush: 0.0020 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5582.8 | uss_MB: 4766.6 | child_rss_MB: 8760.5 | allocated_MB: 611.6 | reserved_MB: 31234.0 | inactive_split_MB: 318.4 +rank5 [2026-10-27 20:14:35,735] INFO : Running validation on rank5 +rank5 [2026-10-27 20:14:35,736] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 20:14:35,736] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 20:14:35,736] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 20:14:35,736] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 20:16:30,818] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 20:16:31,258] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 20:16:55,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 20:16:56,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 20:16:56,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 20:16:57,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 20:16:57,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 20:16:58,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 20:16:58,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 20:16:59,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 20:17:09,287] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 20:17:09,354] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 20:17:09,707] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 20:17:25,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 20:17:25,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 20:17:26,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 20:17:26,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 20:17:27,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 20:17:27,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 20:17:28,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 20:17:28,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 20:17:38,704] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 20:17:39,021] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 20:17:39,450] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 20:17:54,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 20:17:55,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 20:17:55,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 20:17:56,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 20:17:56,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 20:17:57,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 20:17:57,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 20:17:58,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 20:18:08,420] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 20:22:28,768] INFO : Step 61000/100000 rank5 | Train Loss: 12.1474 | Train Loss Unweighted: 4.3289 | Train Loss Components: Classification: 0.0110 | Classification_binary: 3.6236 | Regression_cos_phi: 0.0523 | Regression_energy: 0.2909 | Regression_eta: 0.0654 | Regression_pt: 0.2368 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.6358 | physical_nopu: 4.4439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3960 | Classification_binary: 3.6687 | Regression_cos_phi: 1.6931 | Regression_energy: 1.6164 | Regression_eta: 0.8753 | Regression_pt: 1.7686 | Regression_sin_phi: 1.8187 | LR: 1.59e-04 | DataLoad Time: 0.0065s | Model Forward Time: 3.4741s | Phase Times: max_pre_step_barrier: 0.0115 | max_batch_to_device: 0.0009 | max_forward: 0.2092 | max_loss: 0.0037 | max_backward: 0.1501 | max_optimizer_step: 0.2965 | max_tensorboard_logging: 1.8725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6817.2 | uss_MB: 6009.9 | child_rss_MB: 8755.0 | allocated_MB: 607.8 | reserved_MB: 28822.0 | inactive_split_MB: 222.9 +rank5 [2026-10-27 20:26:14,158] INFO : Step 62000/100000 rank5 | Train Loss: 11.7977 | Train Loss Unweighted: 3.6294 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9084 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2778 | Regression_eta: 0.1039 | Regression_pt: 0.2286 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6417 | physical_nopu: 4.4459 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4890 | Classification_binary: 2.8240 | Regression_cos_phi: 1.7201 | Regression_energy: 1.3702 | Regression_eta: 1.4597 | Regression_pt: 1.5036 | Regression_sin_phi: 1.8363 | LR: 1.54e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.7423s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0033 | max_forward: 0.2219 | max_loss: 0.0037 | max_backward: 0.2965 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1292 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1278 | Memory: rss_MB: 6812.4 | uss_MB: 6017.7 | child_rss_MB: 8760.3 | allocated_MB: 618.8 | reserved_MB: 28822.0 | inactive_split_MB: 888.0 +rank5 [2026-10-27 20:30:05,877] INFO : Step 63000/100000 rank5 | Train Loss: 12.5071 | Train Loss Unweighted: 4.3613 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5997 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3200 | Regression_eta: 0.0719 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6399 | physical_nopu: 4.4447 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2632 | Classification_binary: 3.6118 | Regression_cos_phi: 1.9540 | Regression_energy: 2.0161 | Regression_eta: 1.1288 | Regression_pt: 2.2115 | Regression_sin_phi: 2.3709 | LR: 1.49e-04 | DataLoad Time: 0.0021s | Model Forward Time: 11.0662s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1169 | max_loss: 0.0157 | max_backward: 0.2882 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 9.0944 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6806.2 | uss_MB: 4694.0 | child_rss_MB: 8747.4 | allocated_MB: 610.4 | reserved_MB: 28822.0 | inactive_split_MB: 261.6 +rank5 [2026-10-27 20:33:53,443] INFO : Step 64000/100000 rank5 | Train Loss: 14.0388 | Train Loss Unweighted: 4.5035 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.4200 | Regression_cos_phi: 0.0430 | Regression_energy: 0.4752 | Regression_eta: 0.0663 | Regression_pt: 0.4376 | Regression_sin_phi: 0.0442 | Group Train Loss Unweighted (window mean): physical_pu: 3.6357 | physical_nopu: 4.4395 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5058 | Classification_binary: 3.3557 | Regression_cos_phi: 1.5294 | Regression_energy: 2.5959 | Regression_eta: 0.8529 | Regression_pt: 3.1334 | Regression_sin_phi: 1.9038 | LR: 1.43e-04 | DataLoad Time: 0.0020s | Model Forward Time: 6.4871s | Phase Times: max_pre_step_barrier: 0.1731 | max_batch_to_device: 0.0118 | max_forward: 0.1219 | max_loss: 0.0038 | max_backward: 0.1121 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 5.0441 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5600.0 | uss_MB: 4820.3 | child_rss_MB: 8744.3 | allocated_MB: 611.1 | reserved_MB: 29052.0 | inactive_split_MB: 372.9 +rank5 [2026-10-27 20:37:48,516] INFO : Step 65000/100000 rank5 | Train Loss: 16.3374 | Train Loss Unweighted: 4.8919 | Train Loss Components: Classification: 0.0142 | Classification_binary: 3.4871 | Regression_cos_phi: 0.0500 | Regression_energy: 0.6360 | Regression_eta: 0.0628 | Regression_pt: 0.5943 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4335 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4332 | Classification_binary: 3.5653 | Regression_cos_phi: 1.8246 | Regression_energy: 6.4310 | Regression_eta: 1.0865 | Regression_pt: 8.1976 | Regression_sin_phi: 2.0975 | LR: 1.38e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.4885s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.2167 | max_loss: 0.0058 | max_backward: 0.2407 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 1.9060 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5594.0 | uss_MB: 4820.8 | child_rss_MB: 8750.8 | allocated_MB: 609.7 | reserved_MB: 29052.0 | inactive_split_MB: 236.9 +rank5 [2026-10-27 20:41:37,390] INFO : Step 66000/100000 rank5 | Train Loss: 12.8848 | Train Loss Unweighted: 4.2935 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4715 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3451 | Regression_eta: 0.0693 | Regression_pt: 0.2907 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.6349 | physical_nopu: 4.4348 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4711 | Classification_binary: 3.4663 | Regression_cos_phi: 2.3416 | Regression_energy: 1.6442 | Regression_eta: 0.8342 | Regression_pt: 1.9339 | Regression_sin_phi: 2.8089 | LR: 1.33e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.1925s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0854 | max_loss: 0.1083 | max_backward: 0.2178 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.9218 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5582.1 | uss_MB: 4820.7 | child_rss_MB: 8750.8 | allocated_MB: 611.4 | reserved_MB: 29052.0 | inactive_split_MB: 254.2 +rank5 [2026-10-27 20:45:24,537] INFO : Step 67000/100000 rank5 | Train Loss: 14.2125 | Train Loss Unweighted: 4.5026 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.4493 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4566 | Regression_eta: 0.0671 | Regression_pt: 0.4150 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6352 | physical_nopu: 4.4350 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4833 | Classification_binary: 3.3792 | Regression_cos_phi: 1.7428 | Regression_energy: 2.0465 | Regression_eta: 0.7237 | Regression_pt: 2.5843 | Regression_sin_phi: 1.7949 | LR: 1.28e-04 | DataLoad Time: 0.0029s | Model Forward Time: 3.0116s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0901 | max_loss: 0.0487 | max_backward: 0.1449 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 1.8090 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5575.1 | uss_MB: 4820.8 | child_rss_MB: 8750.2 | allocated_MB: 607.1 | reserved_MB: 29260.0 | inactive_split_MB: 246.6 +rank5 [2026-10-27 20:49:05,126] INFO : Step 68000/100000 rank5 | Train Loss: 12.0487 | Train Loss Unweighted: 4.2633 | Train Loss Components: Classification: 0.0107 | Classification_binary: 3.5135 | Regression_cos_phi: 0.0466 | Regression_energy: 0.3165 | Regression_eta: 0.0645 | Regression_pt: 0.2647 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4387 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3001 | Classification_binary: 3.4584 | Regression_cos_phi: 2.0439 | Regression_energy: 1.8560 | Regression_eta: 1.3776 | Regression_pt: 1.9884 | Regression_sin_phi: 2.3097 | LR: 1.24e-04 | DataLoad Time: 0.0026s | Model Forward Time: 2.8888s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0010 | max_forward: 0.0879 | max_loss: 0.0120 | max_backward: 0.1069 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.7176 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5570.9 | uss_MB: 4820.5 | child_rss_MB: 8753.3 | allocated_MB: 609.5 | reserved_MB: 29260.0 | inactive_split_MB: 225.9 +rank5 [2026-10-27 20:52:47,283] INFO : Step 69000/100000 rank5 | Train Loss: 13.1377 | Train Loss Unweighted: 4.3133 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4496 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3632 | Regression_eta: 0.0778 | Regression_pt: 0.3079 | Regression_sin_phi: 0.0541 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4321 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5063 | Classification_binary: 3.5162 | Regression_cos_phi: 1.8760 | Regression_energy: 1.7244 | Regression_eta: 1.5954 | Regression_pt: 2.0848 | Regression_sin_phi: 1.9933 | LR: 1.19e-04 | DataLoad Time: 0.0014s | Model Forward Time: 4.9130s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.1207 | max_loss: 0.0509 | max_backward: 0.2012 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 3.5532 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 5571.9 | uss_MB: 4820.8 | child_rss_MB: 8755.8 | allocated_MB: 608.0 | reserved_MB: 29260.0 | inactive_split_MB: 256.4 +rank5 [2026-10-27 20:56:27,513] INFO : Step 70000/100000 rank5 | Train Loss: 15.8477 | Train Loss Unweighted: 4.7317 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3873 | Regression_cos_phi: 0.0498 | Regression_energy: 0.6015 | Regression_eta: 0.0629 | Regression_pt: 0.5683 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6315 | physical_nopu: 4.4280 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4793 | Classification_binary: 3.3789 | Regression_cos_phi: 2.1417 | Regression_energy: 3.5904 | Regression_eta: 1.1894 | Regression_pt: 4.5973 | Regression_sin_phi: 2.0148 | LR: 1.14e-04 | DataLoad Time: 0.0071s | Model Forward Time: 2.9280s | Phase Times: max_pre_step_barrier: 0.0198 | max_batch_to_device: 0.0007 | max_forward: 0.1178 | max_loss: 0.0036 | max_backward: 0.1081 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.7610 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5571.5 | uss_MB: 4819.7 | child_rss_MB: 8751.0 | allocated_MB: 610.3 | reserved_MB: 30150.0 | inactive_split_MB: 260.3 +rank5 [2026-10-27 20:56:28,458] INFO : Running validation on rank5 +rank5 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 20:58:25,954] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 20:58:26,360] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 20:58:52,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 20:58:52,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 20:58:53,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 20:58:53,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 20:58:54,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 20:58:54,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 20:58:55,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 20:58:55,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 20:59:05,801] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 20:59:05,947] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 20:59:06,379] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 20:59:21,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 20:59:21,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 20:59:22,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 20:59:22,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 20:59:23,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 20:59:23,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 20:59:24,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 20:59:24,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 20:59:34,955] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 20:59:35,862] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 20:59:36,279] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 20:59:51,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 20:59:51,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 20:59:52,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 20:59:52,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 20:59:53,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 20:59:53,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 20:59:54,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 20:59:54,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 21:00:04,669] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 21:04:25,115] INFO : Step 71000/100000 rank5 | Train Loss: 12.0110 | Train Loss Unweighted: 3.6539 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9046 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2881 | Regression_eta: 0.1063 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6306 | physical_nopu: 4.4256 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4703 | Classification_binary: 2.7999 | Regression_cos_phi: 1.7298 | Regression_energy: 1.4924 | Regression_eta: 1.6352 | Regression_pt: 1.6984 | Regression_sin_phi: 1.8998 | LR: 1.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.4581s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0035 | max_forward: 0.1952 | max_loss: 0.0038 | max_backward: 0.3195 | max_optimizer_step: 0.3092 | max_tensorboard_logging: 1.7411 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6798.7 | uss_MB: 6058.3 | child_rss_MB: 8722.6 | allocated_MB: 623.5 | reserved_MB: 27422.0 | inactive_split_MB: 864.2 +rank5 [2026-10-27 21:08:01,993] INFO : Step 72000/100000 rank5 | Train Loss: 15.2878 | Train Loss Unweighted: 4.6447 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.3643 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5735 | Regression_eta: 0.0647 | Regression_pt: 0.5353 | Regression_sin_phi: 0.0449 | Group Train Loss Unweighted (window mean): physical_pu: 3.6275 | physical_nopu: 4.4239 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4584 | Classification_binary: 3.1555 | Regression_cos_phi: 2.1100 | Regression_energy: 4.1208 | Regression_eta: 0.7883 | Regression_pt: 5.2187 | Regression_sin_phi: 1.8037 | LR: 1.05e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.5452s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0232 | max_forward: 0.1174 | max_loss: 0.0036 | max_backward: 0.0921 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 1.2606 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5641.1 | uss_MB: 4896.7 | child_rss_MB: 8682.2 | allocated_MB: 607.6 | reserved_MB: 28586.0 | inactive_split_MB: 303.9 +rank5 [2026-10-27 21:11:50,556] INFO : Step 73000/100000 rank5 | Train Loss: 12.7380 | Train Loss Unweighted: 4.2392 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.4842 | Regression_cos_phi: 0.0575 | Regression_energy: 0.3093 | Regression_eta: 0.0643 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.6309 | physical_nopu: 4.4270 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2887 | Classification_binary: 3.5165 | Regression_cos_phi: 1.7935 | Regression_energy: 1.5712 | Regression_eta: 0.8619 | Regression_pt: 1.8218 | Regression_sin_phi: 1.9729 | LR: 1.01e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.4474s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0985 | max_loss: 0.0632 | max_backward: 0.1654 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2139 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5637.1 | uss_MB: 4897.0 | child_rss_MB: 8681.3 | allocated_MB: 607.0 | reserved_MB: 28586.0 | inactive_split_MB: 227.5 +rank5 [2026-10-27 21:15:34,268] INFO : Step 74000/100000 rank5 | Train Loss: 12.5735 | Train Loss Unweighted: 4.2478 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4076 | Regression_cos_phi: 0.0429 | Regression_energy: 0.3533 | Regression_eta: 0.0699 | Regression_pt: 0.3116 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6307 | physical_nopu: 4.4218 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4584 | Classification_binary: 3.2838 | Regression_cos_phi: 1.6070 | Regression_energy: 2.1250 | Regression_eta: 0.9723 | Regression_pt: 2.4314 | Regression_sin_phi: 2.0249 | LR: 9.68e-05 | DataLoad Time: 0.0023s | Model Forward Time: 2.5464s | Phase Times: max_pre_step_barrier: 0.1687 | max_batch_to_device: 0.0008 | max_forward: 0.0990 | max_loss: 0.0516 | max_backward: 0.1766 | max_optimizer_step: 0.1812 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5626.2 | uss_MB: 4897.3 | child_rss_MB: 8634.5 | allocated_MB: 608.6 | reserved_MB: 28586.0 | inactive_split_MB: 285.4 +rank5 [2026-10-27 21:19:21,651] INFO : Step 75000/100000 rank5 | Train Loss: 15.1687 | Train Loss Unweighted: 4.6098 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.3751 | Regression_cos_phi: 0.0492 | Regression_energy: 0.5530 | Regression_eta: 0.0638 | Regression_pt: 0.5060 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4206 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4564 | Classification_binary: 3.5360 | Regression_cos_phi: 2.0577 | Regression_energy: 2.4602 | Regression_eta: 0.8532 | Regression_pt: 2.8380 | Regression_sin_phi: 1.6949 | LR: 9.27e-05 | DataLoad Time: 0.0020s | Model Forward Time: 2.4837s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1026 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.2820 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5619.2 | uss_MB: 4896.7 | child_rss_MB: 8602.7 | allocated_MB: 610.1 | reserved_MB: 28586.0 | inactive_split_MB: 259.7 +rank5 [2026-10-27 21:23:05,981] INFO : Step 76000/100000 rank5 | Train Loss: 12.2943 | Train Loss Unweighted: 4.2773 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5085 | Regression_cos_phi: 0.0476 | Regression_energy: 0.3262 | Regression_eta: 0.0732 | Regression_pt: 0.2631 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6289 | physical_nopu: 4.4209 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3446 | Classification_binary: 3.4744 | Regression_cos_phi: 2.2591 | Regression_energy: 2.2173 | Regression_eta: 2.7411 | Regression_pt: 2.1510 | Regression_sin_phi: 2.3157 | LR: 8.88e-05 | DataLoad Time: 0.0020s | Model Forward Time: 2.6229s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.1085 | max_optimizer_step: 0.1831 | max_tensorboard_logging: 1.4262 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 5617.3 | uss_MB: 4895.6 | child_rss_MB: 8601.6 | allocated_MB: 608.9 | reserved_MB: 28942.0 | inactive_split_MB: 227.1 +rank5 [2026-10-27 21:27:01,963] INFO : Step 77000/100000 rank5 | Train Loss: 13.8477 | Train Loss Unweighted: 4.4496 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4225 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4510 | Regression_eta: 0.0623 | Regression_pt: 0.4032 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6274 | physical_nopu: 4.4166 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4578 | Classification_binary: 3.3665 | Regression_cos_phi: 1.8462 | Regression_energy: 3.0384 | Regression_eta: 1.3220 | Regression_pt: 3.6768 | Regression_sin_phi: 2.0651 | LR: 8.50e-05 | DataLoad Time: 0.0021s | Model Forward Time: 6.1302s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0839 | max_forward: 0.1754 | max_loss: 0.0036 | max_backward: 0.1103 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 4.6772 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1374 | Memory: rss_MB: 5607.3 | uss_MB: 4838.5 | child_rss_MB: 8502.5 | allocated_MB: 611.0 | reserved_MB: 28942.0 | inactive_split_MB: 369.0 +rank5 [2026-10-27 21:30:52,663] INFO : Step 78000/100000 rank5 | Train Loss: 14.1981 | Train Loss Unweighted: 4.6212 | Train Loss Components: Classification: 0.0133 | Classification_binary: 3.5747 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4573 | Regression_eta: 0.0669 | Regression_pt: 0.4114 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.6241 | physical_nopu: 4.4230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4909 | Classification_binary: 3.6236 | Regression_cos_phi: 1.6206 | Regression_energy: 3.3566 | Regression_eta: 0.9823 | Regression_pt: 4.2703 | Regression_sin_phi: 1.7847 | LR: 8.13e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.0988s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1149 | max_loss: 0.0636 | max_backward: 0.1914 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5607.3 | uss_MB: 4897.1 | child_rss_MB: 8495.1 | allocated_MB: 606.6 | reserved_MB: 28942.0 | inactive_split_MB: 309.2 +rank5 [2026-10-27 21:34:49,720] INFO : Step 79000/100000 rank5 | Train Loss: 12.3648 | Train Loss Unweighted: 4.2396 | Train Loss Components: Classification: 0.0145 | Classification_binary: 3.4636 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3226 | Regression_eta: 0.0628 | Regression_pt: 0.2792 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6254 | physical_nopu: 4.4183 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4246 | Classification_binary: 3.4947 | Regression_cos_phi: 1.5889 | Regression_energy: 1.6296 | Regression_eta: 0.9037 | Regression_pt: 1.8144 | Regression_sin_phi: 2.2745 | LR: 7.78e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.3389s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0936 | max_forward: 0.1936 | max_loss: 0.0037 | max_backward: 0.1134 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 2.0079 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 5602.4 | uss_MB: 4897.2 | child_rss_MB: 8483.1 | allocated_MB: 609.2 | reserved_MB: 29222.0 | inactive_split_MB: 324.6 +rank5 [2026-10-27 21:38:37,320] INFO : Step 80000/100000 rank5 | Train Loss: 16.1833 | Train Loss Unweighted: 4.8046 | Train Loss Components: Classification: 0.0155 | Classification_binary: 3.4149 | Regression_cos_phi: 0.0453 | Regression_energy: 0.6189 | Regression_eta: 0.0801 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6233 | physical_nopu: 4.4142 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4060 | Classification_binary: 3.5092 | Regression_cos_phi: 1.8554 | Regression_energy: 4.5845 | Regression_eta: 1.0728 | Regression_pt: 5.9897 | Regression_sin_phi: 1.6690 | LR: 7.44e-05 | DataLoad Time: 0.0102s | Model Forward Time: 3.9236s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0008 | max_forward: 0.1071 | max_loss: 0.0037 | max_backward: 0.1380 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 2.6784 | max_tensorboard_flush: 0.0026 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 5596.4 | uss_MB: 4871.2 | child_rss_MB: 8436.2 | allocated_MB: 606.3 | reserved_MB: 29222.0 | inactive_split_MB: 471.7 +rank5 [2026-10-27 21:38:38,322] INFO : Running validation on rank5 +rank5 [2026-10-27 21:38:38,323] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 21:38:38,323] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 21:38:38,323] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 21:38:38,323] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 21:40:39,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 21:40:39,982] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 21:41:05,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 21:41:06,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 21:41:06,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 21:41:07,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 21:41:07,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 21:41:08,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 21:41:08,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 21:41:09,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 21:41:19,247] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 21:41:19,915] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 21:41:20,296] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 21:41:35,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 21:41:35,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 21:41:36,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 21:41:36,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 21:41:37,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 21:41:37,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 21:41:38,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 21:41:38,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 21:41:48,755] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 21:41:49,545] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 21:41:50,005] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 21:42:04,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 21:42:05,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 21:42:05,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 21:42:06,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 21:42:06,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 21:42:07,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 21:42:07,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 21:42:08,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 21:42:18,384] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 21:46:50,145] INFO : Step 81000/100000 rank5 | Train Loss: 11.7500 | Train Loss Unweighted: 3.6065 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.8934 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2712 | Regression_eta: 0.1074 | Regression_pt: 0.2244 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4130 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4802 | Classification_binary: 2.9131 | Regression_cos_phi: 1.8339 | Regression_energy: 1.4541 | Regression_eta: 1.6997 | Regression_pt: 1.5897 | Regression_sin_phi: 1.9551 | LR: 7.11e-05 | DataLoad Time: 0.0019s | Model Forward Time: 8.6909s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0975 | max_forward: 0.3237 | max_loss: 0.0036 | max_backward: 0.5496 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 6.3197 | max_tensorboard_flush: 0.0011 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6963.5 | uss_MB: 4835.8 | child_rss_MB: 8415.8 | allocated_MB: 619.9 | reserved_MB: 28598.0 | inactive_split_MB: 1192.4 +rank5 [2026-10-27 21:50:33,876] INFO : Step 82000/100000 rank5 | Train Loss: 13.6704 | Train Loss Unweighted: 4.5604 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0419 | Regression_energy: 0.4618 | Regression_eta: 0.0599 | Regression_pt: 0.4291 | Regression_sin_phi: 0.0407 | Group Train Loss Unweighted (window mean): physical_pu: 3.6250 | physical_nopu: 4.4152 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5845 | Classification_binary: 3.5608 | Regression_cos_phi: 1.6623 | Regression_energy: 2.1648 | Regression_eta: 0.8512 | Regression_pt: 2.6069 | Regression_sin_phi: 1.8187 | LR: 6.80e-05 | DataLoad Time: 0.0028s | Model Forward Time: 3.1341s | Phase Times: max_pre_step_barrier: 0.1835 | max_batch_to_device: 0.0874 | max_forward: 0.1976 | max_loss: 0.0036 | max_backward: 0.1170 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 1.6260 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 5659.6 | uss_MB: 4976.9 | child_rss_MB: 8356.3 | allocated_MB: 607.5 | reserved_MB: 28598.0 | inactive_split_MB: 253.8 +rank5 [2026-10-27 21:54:19,227] INFO : Step 83000/100000 rank5 | Train Loss: 12.0270 | Train Loss Unweighted: 4.2414 | Train Loss Components: Classification: 0.0092 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0499 | Regression_energy: 0.3056 | Regression_eta: 0.0671 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4141 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2944 | Classification_binary: 3.5013 | Regression_cos_phi: 1.7974 | Regression_energy: 1.7468 | Regression_eta: 1.0958 | Regression_pt: 1.8631 | Regression_sin_phi: 2.0076 | LR: 6.51e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.7069s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1966 | max_loss: 0.0138 | max_backward: 0.1124 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 2.4026 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 5659.6 | uss_MB: 4977.1 | child_rss_MB: 8350.2 | allocated_MB: 609.6 | reserved_MB: 28598.0 | inactive_split_MB: 289.5 +rank5 [2026-10-27 21:58:08,515] INFO : Step 84000/100000 rank5 | Train Loss: 12.5172 | Train Loss Unweighted: 4.2412 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.4173 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3464 | Regression_eta: 0.0637 | Regression_pt: 0.3052 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4149 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4775 | Classification_binary: 3.5683 | Regression_cos_phi: 1.4855 | Regression_energy: 1.4630 | Regression_eta: 0.8058 | Regression_pt: 1.7370 | Regression_sin_phi: 1.6628 | LR: 6.23e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.2648s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1103 | max_loss: 0.0037 | max_backward: 0.1159 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 2.0319 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5655.6 | uss_MB: 4977.8 | child_rss_MB: 8312.0 | allocated_MB: 605.9 | reserved_MB: 28598.0 | inactive_split_MB: 208.8 +rank5 [2026-10-27 22:01:46,561] INFO : Step 85000/100000 rank5 | Train Loss: 15.2814 | Train Loss Unweighted: 4.6863 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.3811 | Regression_cos_phi: 0.0432 | Regression_energy: 0.5842 | Regression_eta: 0.0608 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6173 | physical_nopu: 4.4078 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5133 | Classification_binary: 3.5014 | Regression_cos_phi: 1.1124 | Regression_energy: 2.4555 | Regression_eta: 1.0249 | Regression_pt: 3.1529 | Regression_sin_phi: 1.7257 | LR: 5.96e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.0838s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.0993 | max_loss: 0.0558 | max_backward: 0.1762 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 1.8022 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 5645.3 | uss_MB: 4977.4 | child_rss_MB: 8242.1 | allocated_MB: 606.6 | reserved_MB: 28598.0 | inactive_split_MB: 257.4 +rank5 [2026-10-27 22:05:28,665] INFO : Step 86000/100000 rank5 | Train Loss: 12.8074 | Train Loss Unweighted: 4.3373 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.5785 | Regression_cos_phi: 0.0543 | Regression_energy: 0.3116 | Regression_eta: 0.0747 | Regression_pt: 0.2515 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.6255 | physical_nopu: 4.4086 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3191 | Classification_binary: 3.5323 | Regression_cos_phi: 2.3101 | Regression_energy: 1.4476 | Regression_eta: 0.9745 | Regression_pt: 1.6673 | Regression_sin_phi: 2.2936 | LR: 5.71e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.0434s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1215 | max_loss: 0.0037 | max_backward: 0.1229 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 1.6761 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5636.3 | uss_MB: 4977.9 | child_rss_MB: 8180.8 | allocated_MB: 608.3 | reserved_MB: 29688.0 | inactive_split_MB: 271.8 +rank5 [2026-10-27 22:09:22,620] INFO : Step 87000/100000 rank5 | Train Loss: 12.9093 | Train Loss Unweighted: 4.3003 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.3807 | Regression_cos_phi: 0.0447 | Regression_energy: 0.4014 | Regression_eta: 0.0615 | Regression_pt: 0.3523 | Regression_sin_phi: 0.0424 | Group Train Loss Unweighted (window mean): physical_pu: 3.6211 | physical_nopu: 4.4120 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5190 | Classification_binary: 3.4189 | Regression_cos_phi: 1.5888 | Regression_energy: 1.8715 | Regression_eta: 0.9622 | Regression_pt: 2.1450 | Regression_sin_phi: 1.6732 | LR: 5.48e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.0852s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0203 | max_forward: 0.1299 | max_loss: 0.0038 | max_backward: 0.1175 | max_optimizer_step: 0.2192 | max_tensorboard_logging: 1.8010 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5636.1 | uss_MB: 4575.9 | child_rss_MB: 8182.7 | allocated_MB: 607.0 | reserved_MB: 29688.0 | inactive_split_MB: 231.0 +rank5 [2026-10-27 22:13:02,799] INFO : Step 88000/100000 rank5 | Train Loss: 15.2369 | Train Loss Unweighted: 4.6456 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4525 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5240 | Regression_eta: 0.0665 | Regression_pt: 0.4840 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6192 | physical_nopu: 4.4119 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4248 | Classification_binary: 3.5291 | Regression_cos_phi: 1.3370 | Regression_energy: 2.3214 | Regression_eta: 0.7806 | Regression_pt: 2.8226 | Regression_sin_phi: 1.8071 | LR: 5.26e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.1171s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0007 | max_forward: 0.1127 | max_loss: 0.0236 | max_backward: 0.1542 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 1.7956 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5636.1 | uss_MB: 4977.3 | child_rss_MB: 8181.7 | allocated_MB: 609.0 | reserved_MB: 29688.0 | inactive_split_MB: 221.0 +rank5 [2026-10-27 22:16:52,919] INFO : Step 89000/100000 rank5 | Train Loss: 12.8609 | Train Loss Unweighted: 4.2765 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.4665 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3343 | Regression_eta: 0.0681 | Regression_pt: 0.2896 | Regression_sin_phi: 0.0555 | Group Train Loss Unweighted (window mean): physical_pu: 3.6177 | physical_nopu: 4.4110 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3767 | Classification_binary: 3.3255 | Regression_cos_phi: 1.4856 | Regression_energy: 1.7815 | Regression_eta: 1.0012 | Regression_pt: 1.9461 | Regression_sin_phi: 1.9617 | LR: 5.06e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.1963s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1194 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 1.9452 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 5632.5 | uss_MB: 4977.7 | child_rss_MB: 8137.7 | allocated_MB: 612.6 | reserved_MB: 30628.0 | inactive_split_MB: 309.7 +rank5 [2026-10-27 22:20:43,711] INFO : Step 90000/100000 rank5 | Train Loss: 11.6102 | Train Loss Unweighted: 3.6127 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9177 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2661 | Regression_eta: 0.1040 | Regression_pt: 0.2155 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4109 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4684 | Classification_binary: 2.9279 | Regression_cos_phi: 1.7291 | Regression_energy: 1.4273 | Regression_eta: 1.5469 | Regression_pt: 1.5563 | Regression_sin_phi: 1.8468 | LR: 4.88e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.5773s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0110 | max_forward: 0.1856 | max_loss: 0.0145 | max_backward: 0.4526 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 1.8620 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 5624.5 | uss_MB: 4985.7 | child_rss_MB: 8062.4 | allocated_MB: 617.2 | reserved_MB: 30628.0 | inactive_split_MB: 962.2 +rank5 [2026-10-27 22:20:44,690] INFO : Running validation on rank5 +rank5 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 22:22:45,204] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 22:22:45,606] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 22:23:10,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 22:23:10,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 22:23:11,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 22:23:11,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 22:23:12,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 22:23:12,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 22:23:13,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 22:23:14,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 22:23:24,019] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 22:23:24,366] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 22:23:24,734] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 22:23:40,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 22:23:40,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 22:23:41,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 22:23:41,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 22:23:42,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 22:23:42,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 22:23:43,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 22:23:43,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 22:23:53,750] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 22:23:53,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 22:23:54,273] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 22:24:09,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 22:24:09,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 22:24:10,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 22:24:10,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 22:24:11,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 22:24:11,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 22:24:12,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 22:24:12,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 22:24:22,902] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 22:28:41,211] INFO : Step 91000/100000 rank5 | Train Loss: 12.9063 | Train Loss Unweighted: 4.3371 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5236 | Regression_cos_phi: 0.0531 | Regression_energy: 0.3431 | Regression_eta: 0.0707 | Regression_pt: 0.2829 | Regression_sin_phi: 0.0512 | Group Train Loss Unweighted (window mean): physical_pu: 3.6180 | physical_nopu: 4.4087 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3561 | Classification_binary: 3.3948 | Regression_cos_phi: 1.9005 | Regression_energy: 1.5006 | Regression_eta: 0.9770 | Regression_pt: 1.6111 | Regression_sin_phi: 1.7518 | LR: 4.71e-05 | DataLoad Time: 0.0018s | Model Forward Time: 4.9000s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0055 | max_forward: 0.1521 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 3.4850 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7066.6 | uss_MB: 6432.9 | child_rss_MB: 8020.4 | allocated_MB: 607.8 | reserved_MB: 28796.0 | inactive_split_MB: 320.2 +rank5 [2026-10-27 22:32:23,431] INFO : Step 92000/100000 rank5 | Train Loss: 15.4341 | Train Loss Unweighted: 4.7247 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4285 | Regression_cos_phi: 0.0457 | Regression_energy: 0.5811 | Regression_eta: 0.0735 | Regression_pt: 0.5375 | Regression_sin_phi: 0.0425 | Group Train Loss Unweighted (window mean): physical_pu: 3.6150 | physical_nopu: 4.4032 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4507 | Classification_binary: 3.5013 | Regression_cos_phi: 1.7924 | Regression_energy: 2.0954 | Regression_eta: 1.1787 | Regression_pt: 2.4577 | Regression_sin_phi: 1.5206 | LR: 4.57e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.3950s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.2696 | max_loss: 0.0037 | max_backward: 0.1715 | max_optimizer_step: 0.3237 | max_tensorboard_logging: 1.7768 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7051.6 | uss_MB: 6432.9 | child_rss_MB: 7931.4 | allocated_MB: 609.3 | reserved_MB: 28796.0 | inactive_split_MB: 222.8 +rank5 [2026-10-27 22:36:08,796] INFO : Step 93000/100000 rank5 | Train Loss: 13.9744 | Train Loss Unweighted: 4.4854 | Train Loss Components: Classification: 0.0118 | Classification_binary: 3.5124 | Regression_cos_phi: 0.0578 | Regression_energy: 0.4245 | Regression_eta: 0.0652 | Regression_pt: 0.3624 | Regression_sin_phi: 0.0513 | Group Train Loss Unweighted (window mean): physical_pu: 3.6205 | physical_nopu: 4.4075 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2924 | Classification_binary: 3.6015 | Regression_cos_phi: 1.8953 | Regression_energy: 1.8725 | Regression_eta: 1.1708 | Regression_pt: 1.8863 | Regression_sin_phi: 1.7755 | LR: 4.43e-05 | DataLoad Time: 0.0021s | Model Forward Time: 2.9851s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1098 | max_loss: 0.0036 | max_backward: 0.1173 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 1.7709 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5706.9 | uss_MB: 5092.9 | child_rss_MB: 7894.0 | allocated_MB: 606.4 | reserved_MB: 30394.0 | inactive_split_MB: 244.2 +rank5 [2026-10-27 22:39:53,036] INFO : Step 94000/100000 rank5 | Train Loss: 12.4397 | Train Loss Unweighted: 4.2538 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4230 | Regression_cos_phi: 0.0415 | Regression_energy: 0.3513 | Regression_eta: 0.0702 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6164 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5042 | Classification_binary: 3.6080 | Regression_cos_phi: 1.8718 | Regression_energy: 1.6775 | Regression_eta: 1.2165 | Regression_pt: 1.9736 | Regression_sin_phi: 2.2702 | LR: 4.32e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.1518s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0009 | max_forward: 0.1540 | max_loss: 0.0036 | max_backward: 0.2309 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 1.7723 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 5695.9 | uss_MB: 4753.0 | child_rss_MB: 7806.9 | allocated_MB: 607.1 | reserved_MB: 30394.0 | inactive_split_MB: 232.2 +rank5 [2026-10-27 22:43:36,954] INFO : Step 95000/100000 rank5 | Train Loss: 14.9177 | Train Loss Unweighted: 4.6108 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3911 | Regression_cos_phi: 0.0436 | Regression_energy: 0.5612 | Regression_eta: 0.0598 | Regression_pt: 0.4905 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6121 | physical_nopu: 4.4076 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4953 | Classification_binary: 3.5445 | Regression_cos_phi: 1.6555 | Regression_energy: 2.3978 | Regression_eta: 0.6771 | Regression_pt: 2.8874 | Regression_sin_phi: 1.7667 | LR: 4.22e-05 | DataLoad Time: 0.0017s | Model Forward Time: 3.0355s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1111 | max_loss: 0.0199 | max_backward: 0.1395 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.7978 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5695.9 | uss_MB: 5093.1 | child_rss_MB: 7811.5 | allocated_MB: 609.0 | reserved_MB: 30394.0 | inactive_split_MB: 285.3 +rank5 [2026-10-27 22:47:14,355] INFO : Step 96000/100000 rank5 | Train Loss: 12.4738 | Train Loss Unweighted: 4.2244 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.4884 | Regression_cos_phi: 0.0540 | Regression_energy: 0.2967 | Regression_eta: 0.0804 | Regression_pt: 0.2429 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6137 | physical_nopu: 4.4081 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3407 | Classification_binary: 3.3100 | Regression_cos_phi: 1.5527 | Regression_energy: 1.4682 | Regression_eta: 1.0365 | Regression_pt: 1.5255 | Regression_sin_phi: 1.9280 | LR: 4.14e-05 | DataLoad Time: 0.0027s | Model Forward Time: 3.1602s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0007 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.2003 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 1.8675 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5695.9 | uss_MB: 5093.2 | child_rss_MB: 7804.9 | allocated_MB: 608.1 | reserved_MB: 30394.0 | inactive_split_MB: 259.4 +rank5 [2026-10-27 22:50:52,983] INFO : Step 97000/100000 rank5 | Train Loss: 12.7922 | Train Loss Unweighted: 4.2852 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.4355 | Regression_cos_phi: 0.0455 | Regression_energy: 0.3604 | Regression_eta: 0.0703 | Regression_pt: 0.3091 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6136 | physical_nopu: 4.4015 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4157 | Classification_binary: 3.4860 | Regression_cos_phi: 1.8955 | Regression_energy: 2.2293 | Regression_eta: 0.8417 | Regression_pt: 2.5098 | Regression_sin_phi: 2.0628 | LR: 4.08e-05 | DataLoad Time: 0.0013s | Model Forward Time: 3.0117s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1110 | max_loss: 0.0174 | max_backward: 0.1239 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 1.7282 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0908 | Memory: rss_MB: 5687.4 | uss_MB: 5092.4 | child_rss_MB: 7746.9 | allocated_MB: 607.2 | reserved_MB: 30394.0 | inactive_split_MB: 265.3 +rank5 [2026-10-27 22:54:49,600] INFO : Step 98000/100000 rank5 | Train Loss: 13.8508 | Train Loss Unweighted: 4.4512 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3736 | Regression_cos_phi: 0.0443 | Regression_energy: 0.4792 | Regression_eta: 0.0606 | Regression_pt: 0.4354 | Regression_sin_phi: 0.0433 | Group Train Loss Unweighted (window mean): physical_pu: 3.6169 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4512 | Classification_binary: 3.2262 | Regression_cos_phi: 1.2665 | Regression_energy: 3.1483 | Regression_eta: 0.7624 | Regression_pt: 3.8717 | Regression_sin_phi: 1.3123 | LR: 4.04e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.0658s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0119 | max_forward: 0.1317 | max_loss: 0.0036 | max_backward: 0.1246 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.7312 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0878 | Memory: rss_MB: 5687.4 | uss_MB: 5093.6 | child_rss_MB: 7731.7 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 292.7 +rank5 [2026-10-27 22:58:30,099] INFO : Step 99000/100000 rank5 | Train Loss: 12.5729 | Train Loss Unweighted: 4.3128 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5536 | Regression_cos_phi: 0.0519 | Regression_energy: 0.3097 | Regression_eta: 0.0644 | Regression_pt: 0.2679 | Regression_sin_phi: 0.0529 | Group Train Loss Unweighted (window mean): physical_pu: 3.6154 | physical_nopu: 4.4035 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3191 | Classification_binary: 3.4781 | Regression_cos_phi: 2.2142 | Regression_energy: 1.4381 | Regression_eta: 0.8364 | Regression_pt: 1.6941 | Regression_sin_phi: 1.9472 | LR: 4.01e-05 | DataLoad Time: 0.0113s | Model Forward Time: 3.0443s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0008 | max_forward: 0.1158 | max_loss: 0.0113 | max_backward: 0.1168 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 1.7726 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 5687.4 | uss_MB: 5093.5 | child_rss_MB: 7727.1 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 283.7 +rank5 [2026-10-27 23:02:17,099] INFO : Step 100000/100000 rank5 | Train Loss: 11.9488 | Train Loss Unweighted: 3.6450 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9057 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2836 | Regression_eta: 0.1038 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6201 | physical_nopu: 4.4070 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5305 | Classification_binary: 2.9072 | Regression_cos_phi: 1.7385 | Regression_energy: 1.5360 | Regression_eta: 1.6208 | Regression_pt: 1.7066 | Regression_sin_phi: 1.9746 | LR: 4.00e-05 | DataLoad Time: 0.0029s | Model Forward Time: 6.3807s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0031 | max_forward: 0.2860 | max_loss: 0.0036 | max_backward: 0.4897 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 4.4601 | max_tensorboard_flush: 0.0009 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5687.2 | uss_MB: 4556.1 | child_rss_MB: 7711.3 | allocated_MB: 618.6 | reserved_MB: 30394.0 | inactive_split_MB: 1112.1 +rank5 [2026-10-27 23:02:18,080] INFO : Running validation on rank5 +rank5 [2026-10-27 23:02:18,081] INFO : ResumableSampler physical_pu:valid resetting +rank5 [2026-10-27 23:02:18,081] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank5 [2026-10-27 23:02:18,081] INFO : ResumableSampler physical_nopu:valid resetting +rank5 [2026-10-27 23:02:18,081] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank5 [2026-10-27 23:04:23,624] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 23:04:24,088] INFO : Running predictions on cms_pf_ttbar +rank5 [2026-10-27 23:04:49,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_5_0.parquet +rank5 [2026-10-27 23:04:50,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_5_1.parquet +rank5 [2026-10-27 23:04:50,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_5_2.parquet +rank5 [2026-10-27 23:04:51,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_5_3.parquet +rank5 [2026-10-27 23:04:51,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_5_4.parquet +rank5 [2026-10-27 23:04:52,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_5_5.parquet +rank5 [2026-10-27 23:04:52,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_5_6.parquet +rank5 [2026-10-27 23:04:53,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_5_7.parquet +rank5 [2026-10-27 23:05:03,435] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 23:05:03,812] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 23:05:04,178] INFO : Running predictions on cms_pf_qcd +rank5 [2026-10-27 23:05:19,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_5_0.parquet +rank5 [2026-10-27 23:05:20,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_5_1.parquet +rank5 [2026-10-27 23:05:20,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_5_2.parquet +rank5 [2026-10-27 23:05:21,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_5_3.parquet +rank5 [2026-10-27 23:05:21,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_5_4.parquet +rank5 [2026-10-27 23:05:22,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_5_5.parquet +rank5 [2026-10-27 23:05:22,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_5_6.parquet +rank5 [2026-10-27 23:05:23,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_5_7.parquet +rank5 [2026-10-27 23:05:33,378] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-27 23:05:33,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-27 23:05:34,200] INFO : Running predictions on cms_pf_ztt +rank5 [2026-10-27 23:05:49,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_5_0.parquet +rank5 [2026-10-27 23:05:50,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_5_1.parquet +rank5 [2026-10-27 23:05:50,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_5_2.parquet +rank5 [2026-10-27 23:05:51,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_5_3.parquet +rank5 [2026-10-27 23:05:51,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_5_4.parquet +rank5 [2026-10-27 23:05:52,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_5_5.parquet +rank5 [2026-10-27 23:05:52,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_5_6.parquet +rank5 [2026-10-27 23:05:53,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_5_7.parquet +rank5 [2026-10-27 23:06:03,259] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-27 23:06:03,975] INFO : Training completed. Total time on device 5: 414.791min diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.6 b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.6 new file mode 100644 index 0000000000000000000000000000000000000000..154a7d67a4607d44950e20aef356c38d906707b1 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.6 @@ -0,0 +1,897 @@ +rank6 [2026-10-27 16:10:10,863] INFO : Initializing model with seed=12345; process seed=12351 +rank6 [2026-10-27 16:10:10,864] INFO : configured dtype=torch.bfloat16 for autocast +rank6 [2026-10-27 16:10:35,267] INFO : Instantiating model +rank6 [2026-10-27 16:10:35,267] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank6 [2026-10-27 16:10:35,267] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank6 [2026-10-27 16:10:35,303] INFO : Input encoding initialization took 0.04s +rank6 [2026-10-27 16:10:35,304] INFO : nn0 parameters: 2130944 +rank6 [2026-10-27 16:10:35,304] INFO : Initializing shared backbone layers, num_convs=3 +rank6 [2026-10-27 16:10:35,304] INFO : Initializing backbone layer 0 +rank6 [2026-10-27 16:10:35,350] INFO : Initializing backbone layer 1 +rank6 [2026-10-27 16:10:35,379] INFO : Initializing backbone layer 2 +rank6 [2026-10-27 16:10:35,408] INFO : Backbone initialization took 0.10s +rank6 [2026-10-27 16:10:35,408] INFO : backbone parameters: 15765696 +rank6 [2026-10-27 16:10:35,408] INFO : Initializing output DNNs +rank6 [2026-10-27 16:10:35,442] INFO : Output DNNs initialization took 0.03s +rank6 [2026-10-27 16:10:35,442] INFO : backbone_mode=BackboneMode.SHARED +rank6 [2026-10-27 16:10:35,442] INFO : jagged_attention enabled=False +rank6 [2026-10-27 16:10:35,442] INFO : flash_attn_varlen enabled=False +rank6 [2026-10-27 16:10:35,442] INFO : task_queries enabled=False +rank6 [2026-10-27 16:10:35,442] INFO : classification_norm parameters: 0 +rank6 [2026-10-27 16:10:35,442] INFO : regression_norm parameters: 0 +rank6 [2026-10-27 16:10:35,442] INFO : classification_query parameters: 0 +rank6 [2026-10-27 16:10:35,442] INFO : regression_query parameters: 0 +rank6 [2026-10-27 16:10:35,442] INFO : classification_readout parameters: 0 +rank6 [2026-10-27 16:10:35,442] INFO : regression_readout parameters: 0 +rank6 [2026-10-27 16:10:35,442] INFO : output_mode=elementwise +rank6 [2026-10-27 16:10:35,442] INFO : set_decoder parameters: 0 +rank6 [2026-10-27 16:10:35,442] INFO : nn_binary_particle parameters: 264706 +rank6 [2026-10-27 16:10:35,442] INFO : nn_pid parameters: 268297 +rank6 [2026-10-27 16:10:35,443] INFO : nn_pt parameters: 2113544 +rank6 [2026-10-27 16:10:35,443] INFO : nn_eta parameters: 264706 +rank6 [2026-10-27 16:10:35,443] INFO : nn_sin_phi parameters: 264706 +rank6 [2026-10-27 16:10:35,443] INFO : nn_cos_phi parameters: 264706 +rank6 [2026-10-27 16:10:35,443] INFO : nn_energy parameters: 2113544 +rank6 [2026-10-27 16:10:35,444] INFO : Total MLPF parameters: 23450849 +rank6 [2026-10-27 16:10:35,444] INFO : MLPF __init__ done +rank6 [2026-10-27 16:10:35,444] INFO : Instantiated model +rank6 [2026-10-27 16:10:35,444] INFO : Moving model to device rank=6 +rank6 [2026-10-27 16:10:35,476] INFO : Moved model to device rank=6 +rank6 [2026-10-27 16:10:35,478] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank6 [2026-10-27 16:10:35,478] INFO : Configured model for SyncBatchNorm rank=6 +rank6 [2026-10-27 16:10:36,005] INFO : Configured model for DistributedDataParallel rank=6 +rank6 [2026-10-27 16:10:36,123] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +202 module.nn_energy.nn.7.0.bias ... 0 +203 module.nn_energy.nn.7.2.weight ... 0 +204 module.nn_energy.nn.7.2.bias ... 0 +205 module.nn_energy.nn.7.4.weight ... 0 +206 module.nn_energy.nn.7.4.bias ... 0 + +[207 rows x 3 columns] +rank6 [2026-10-27 16:10:36,123] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x HEPTv2Layer( + embedding_dim=512, num_heads=16, width=512, dropout=0.0, block_size=100, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=512, out_features=512, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=512, out_features=512, bias=False) + (w_k): Linear(in_features=512, out_features=512, bias=False) + (w_v): Linear(in_features=512, out_features=512, bias=False) + (w_rpe): Linear(in_features=10, out_features=512, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=2048, bias=True) + (1): LayerNorm((2048,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=2048, out_features=512, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=512, out_features=2048, bias=False) + (up_proj): Linear(in_features=512, out_features=2048, bias=False) + (down_proj): Linear(in_features=2048, out_features=512, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank6 [2026-10-27 16:10:36,124] INFO : Trainable parameters: 23450849 +rank6 [2026-10-27 16:10:36,124] INFO : Non-trainable parameters: 22386 +rank6 [2026-10-27 16:10:36,124] INFO : Total parameters: 23473235 +rank6 [2026-10-27 16:10:36,126] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 262144 0 + module.backbone.0.attn.out_linear.bias 512 0 + module.backbone.0.attn.e2lsh.alpha 0 1632 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 262144 0 + module.backbone.0.w_k.weight 262144 0 + module.backbone.0.w_v.weight 262144 0 + module.backbone.0.w_rpe.weight 0 5120 + module.backbone.0.w_rpe.bias 0 512 +module.backbone.0.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm2.weight 512 0 + module.backbone.0.q_norm.weight 32 0 + module.backbone.0.k_norm.weight 32 0 + module.backbone.0.ff.gate_proj.weight 1048576 0 + module.backbone.0.ff.up_proj.weight 1048576 0 + module.backbone.0.ff.down_proj.weight 1048576 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 262144 0 + module.backbone.1.attn.out_linear.bias 512 0 + module.backbone.1.attn.e2lsh.alpha 0 1632 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 262144 0 + module.backbone.1.w_k.weight 262144 0 + module.backbone.1.w_v.weight 262144 0 + module.backbone.1.w_rpe.weight 0 5120 + module.backbone.1.w_rpe.bias 0 512 +module.backbone.1.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm2.weight 512 0 + module.backbone.1.q_norm.weight 32 0 + module.backbone.1.k_norm.weight 32 0 + module.backbone.1.ff.gate_proj.weight 1048576 0 + module.backbone.1.ff.up_proj.weight 1048576 0 + module.backbone.1.ff.down_proj.weight 1048576 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 262144 0 + module.backbone.2.attn.out_linear.bias 512 0 + module.backbone.2.attn.e2lsh.alpha 0 1632 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 262144 0 + module.backbone.2.w_k.weight 262144 0 + module.backbone.2.w_v.weight 262144 0 + module.backbone.2.w_rpe.weight 0 5120 + module.backbone.2.w_rpe.bias 0 512 +module.backbone.2.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm2.weight 512 0 + module.backbone.2.q_norm.weight 32 0 + module.backbone.2.k_norm.weight 32 0 + module.backbone.2.ff.gate_proj.weight 1048576 0 + module.backbone.2.ff.up_proj.weight 1048576 0 + module.backbone.2.ff.down_proj.weight 1048576 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank6 [2026-10-27 16:10:36,126] INFO : Getting dataloaders +rank6 [2026-10-27 16:10:36,126] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:43,313] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:44,040] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:44,840] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank6 [2026-10-27 16:10:44,840] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:45,824] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:46,992] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:47,783] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank6 [2026-10-27 16:10:47,784] INFO : Creating WeightedMixIterator with 2 data loaders. +rank6 [2026-10-27 16:10:47,784] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:47,942] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:48,106] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:48,262] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank6 [2026-10-27 16:10:48,263] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:48,429] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:48,595] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:10:48,752] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank6 [2026-10-27 16:10:48,752] INFO : Creating InterleavedIterator with 2 data loaders. +rank6 [2026-10-27 16:11:16,663] INFO : Got dataloaders +rank6 [2026-10-27 16:11:16,663] INFO : loader split=train rank=6 len=232058 +rank6 [2026-10-27 16:11:16,663] INFO : loader split=valid rank=6 len=48 +rank6 [2026-10-27 16:11:16,663] INFO : Starting training from step 1 to 100000 on rank 6 of 8 +rank6 [2026-10-27 16:11:16,663] INFO : Creating train_iterator +rank6 [2026-10-27 16:11:16,663] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank6 [2026-10-27 16:11:30,780] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank6 [2026-10-27 16:11:46,209] INFO : Created train_iterator +rank6 [2026-10-27 16:12:32,259] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.129867712855339, 'Classification': 0.16580097352620216, 'Regression_pt': 0.7168090478330851, 'Regression_eta': 0.335200086296536, 'Regression_sin_phi': 0.1274821354681626, 'Regression_cos_phi': 0.1371471309568733, 'Regression_energy': 0.9554258581250906}, weights={'Classification_binary': 1.0, 'Classification': 30.939912796020508, 'Regression_pt': 7.156533241271973, 'Regression_eta': 15.3038969039917, 'Regression_sin_phi': 40.23989486694336, 'Regression_cos_phi': 37.40412139892578, 'Regression_energy': 5.369194984436035} +rank6 [2026-10-27 16:15:18,322] INFO : Step 1000/100000 rank6 | Train Loss: 16.8632 | Train Loss Unweighted: 5.4934 | Train Loss Components: Classification: 0.0189 | Classification_binary: 4.1833 | Regression_cos_phi: 0.0509 | Regression_energy: 0.5806 | Regression_eta: 0.0776 | Regression_pt: 0.5302 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 5.1301 | physical_nopu: 6.0772 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6836 | Classification_binary: 4.0996 | Regression_cos_phi: 1.8366 | Regression_energy: 2.6486 | Regression_eta: 0.8698 | Regression_pt: 3.2193 | Regression_sin_phi: 2.2786 | LR: 4.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.3142s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0014 | max_forward: 0.0387 | max_loss: 0.0036 | max_backward: 0.0664 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 1.2817 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 4374.6 | uss_MB: 3102.3 | child_rss_MB: 11636.1 | allocated_MB: 609.1 | reserved_MB: 27788.0 | inactive_split_MB: 244.1 +rank6 [2026-10-27 16:18:42,007] INFO : Step 2000/100000 rank6 | Train Loss: 17.7204 | Train Loss Unweighted: 5.4242 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.9240 | Regression_cos_phi: 0.0505 | Regression_energy: 0.6758 | Regression_eta: 0.0665 | Regression_pt: 0.6366 | Regression_sin_phi: 0.0551 | Group Train Loss Unweighted (window mean): physical_pu: 4.2802 | physical_nopu: 5.3470 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6589 | Classification_binary: 3.8349 | Regression_cos_phi: 2.1257 | Regression_energy: 4.2589 | Regression_eta: 1.2632 | Regression_pt: 5.7110 | Regression_sin_phi: 3.1355 | LR: 4.00e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.8063s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1422 | max_loss: 0.0036 | max_backward: 0.1717 | max_optimizer_step: 0.1501 | max_tensorboard_logging: 0.7287 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 4367.5 | uss_MB: 3102.8 | child_rss_MB: 11637.6 | allocated_MB: 607.0 | reserved_MB: 27788.0 | inactive_split_MB: 326.9 +rank6 [2026-10-27 16:22:10,507] INFO : Step 3000/100000 rank6 | Train Loss: 14.3102 | Train Loss Unweighted: 4.8875 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.8970 | Regression_cos_phi: 0.0491 | Regression_energy: 0.4277 | Regression_eta: 0.0716 | Regression_pt: 0.3769 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 4.0705 | physical_nopu: 5.1319 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5391 | Classification_binary: 4.0154 | Regression_cos_phi: 2.4441 | Regression_energy: 2.2490 | Regression_eta: 1.2963 | Regression_pt: 2.6551 | Regression_sin_phi: 2.2763 | LR: 3.99e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.0068s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0782 | max_loss: 0.0037 | max_backward: 0.0505 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 1.0271 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4295.2 | uss_MB: 3034.9 | child_rss_MB: 11625.5 | allocated_MB: 607.6 | reserved_MB: 28576.0 | inactive_split_MB: 261.0 +rank6 [2026-10-27 16:25:44,194] INFO : Step 4000/100000 rank6 | Train Loss: 12.6160 | Train Loss Unweighted: 3.9080 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.1145 | Regression_cos_phi: 0.0484 | Regression_energy: 0.3088 | Regression_eta: 0.1117 | Regression_pt: 0.2597 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.9908 | physical_nopu: 5.0219 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5046 | Classification_binary: 3.0969 | Regression_cos_phi: 1.8568 | Regression_energy: 1.6409 | Regression_eta: 1.8116 | Regression_pt: 1.8281 | Regression_sin_phi: 2.0546 | LR: 3.99e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4272s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0032 | max_forward: 0.1365 | max_loss: 0.0036 | max_backward: 0.2496 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 1.0566 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 4292.4 | uss_MB: 3043.1 | child_rss_MB: 11491.5 | allocated_MB: 617.3 | reserved_MB: 28576.0 | inactive_split_MB: 1049.1 +rank6 [2026-10-27 16:29:29,026] INFO : Step 5000/100000 rank6 | Train Loss: 13.7309 | Train Loss Unweighted: 4.7575 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.8762 | Regression_cos_phi: 0.0508 | Regression_energy: 0.3661 | Regression_eta: 0.0922 | Regression_pt: 0.3105 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.9453 | physical_nopu: 4.9561 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4338 | Classification_binary: 3.9392 | Regression_cos_phi: 2.7012 | Regression_energy: 2.0317 | Regression_eta: 1.2609 | Regression_pt: 2.2101 | Regression_sin_phi: 1.9841 | LR: 3.98e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.0872s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.0008 | max_forward: 0.0371 | max_loss: 0.0036 | max_backward: 0.0568 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 2.1174 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4241.6 | uss_MB: 3036.2 | child_rss_MB: 11146.9 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 248.7 +rank6 [2026-10-27 16:33:03,739] INFO : Step 6000/100000 rank6 | Train Loss: 15.2316 | Train Loss Unweighted: 4.9080 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.7479 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5147 | Regression_eta: 0.0679 | Regression_pt: 0.4609 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.9059 | physical_nopu: 4.9043 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4928 | Classification_binary: 3.6407 | Regression_cos_phi: 2.2099 | Regression_energy: 2.7019 | Regression_eta: 1.5388 | Regression_pt: 3.4246 | Regression_sin_phi: 1.8401 | LR: 3.97e-04 | DataLoad Time: 0.0009s | Model Forward Time: 3.0057s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0421 | max_loss: 0.0036 | max_backward: 0.0620 | max_optimizer_step: 0.1489 | max_tensorboard_logging: 2.0592 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4216.9 | uss_MB: 3036.3 | child_rss_MB: 11044.6 | allocated_MB: 610.3 | reserved_MB: 28576.0 | inactive_split_MB: 254.0 +rank6 [2026-10-27 16:36:49,903] INFO : Step 7000/100000 rank6 | Train Loss: 14.7215 | Train Loss Unweighted: 4.8917 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8534 | Regression_cos_phi: 0.0503 | Regression_energy: 0.4508 | Regression_eta: 0.0791 | Regression_pt: 0.3921 | Regression_sin_phi: 0.0547 | Group Train Loss Unweighted (window mean): physical_pu: 3.8763 | physical_nopu: 4.8627 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2972 | Classification_binary: 4.0540 | Regression_cos_phi: 1.8475 | Regression_energy: 2.3878 | Regression_eta: 1.3523 | Regression_pt: 2.5122 | Regression_sin_phi: 2.3489 | LR: 3.96e-04 | DataLoad Time: 0.0016s | Model Forward Time: 4.5448s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0348 | max_loss: 0.0036 | max_backward: 0.0535 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 3.5409 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4194.7 | uss_MB: 3033.5 | child_rss_MB: 10983.9 | allocated_MB: 609.0 | reserved_MB: 28576.0 | inactive_split_MB: 216.2 +rank6 [2026-10-27 16:40:34,139] INFO : Step 8000/100000 rank6 | Train Loss: 14.2560 | Train Loss Unweighted: 4.8042 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.8045 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4333 | Regression_eta: 0.0601 | Regression_pt: 0.3883 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.8518 | physical_nopu: 4.8294 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6108 | Classification_binary: 3.7072 | Regression_cos_phi: 1.9374 | Regression_energy: 2.3631 | Regression_eta: 0.9474 | Regression_pt: 2.9663 | Regression_sin_phi: 1.7366 | LR: 3.94e-04 | DataLoad Time: 0.0018s | Model Forward Time: 4.8862s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1342 | max_loss: 0.0036 | max_backward: 0.0500 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 3.8645 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4176.9 | uss_MB: 3036.4 | child_rss_MB: 10934.3 | allocated_MB: 606.6 | reserved_MB: 28576.0 | inactive_split_MB: 255.4 +rank6 [2026-10-27 16:44:13,306] INFO : Step 9000/100000 rank6 | Train Loss: 15.9274 | Train Loss Unweighted: 4.9361 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.6437 | Regression_cos_phi: 0.0472 | Regression_energy: 0.5701 | Regression_eta: 0.0733 | Regression_pt: 0.5360 | Regression_sin_phi: 0.0496 | Group Train Loss Unweighted (window mean): physical_pu: 3.8376 | physical_nopu: 4.8057 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5013 | Classification_binary: 3.6225 | Regression_cos_phi: 1.9355 | Regression_energy: 2.2253 | Regression_eta: 1.0019 | Regression_pt: 2.6330 | Regression_sin_phi: 2.2247 | LR: 3.93e-04 | DataLoad Time: 0.0026s | Model Forward Time: 3.2102s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1320 | max_loss: 0.0037 | max_backward: 0.0555 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 2.0797 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 4157.5 | uss_MB: 3035.9 | child_rss_MB: 10877.2 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 321.4 +rank6 [2026-10-27 16:48:05,432] INFO : Step 10000/100000 rank6 | Train Loss: 13.0514 | Train Loss Unweighted: 4.5682 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.7678 | Regression_cos_phi: 0.0520 | Regression_energy: 0.3372 | Regression_eta: 0.0736 | Regression_pt: 0.2745 | Regression_sin_phi: 0.0521 | Group Train Loss Unweighted (window mean): physical_pu: 3.8237 | physical_nopu: 4.7797 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2791 | Classification_binary: 3.8531 | Regression_cos_phi: 2.2141 | Regression_energy: 1.6545 | Regression_eta: 1.3539 | Regression_pt: 1.5764 | Regression_sin_phi: 1.7116 | LR: 3.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.2566s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.0379 | max_loss: 0.0040 | max_backward: 0.0556 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 2.2516 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4149.4 | uss_MB: 3036.1 | child_rss_MB: 10872.4 | allocated_MB: 607.9 | reserved_MB: 28576.0 | inactive_split_MB: 258.7 +rank6 [2026-10-27 16:48:06,334] INFO : Running validation on rank6 +rank6 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 16:50:06,286] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:50:06,771] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 16:50:34,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 16:50:35,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 16:50:36,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 16:50:36,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 16:50:37,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 16:50:37,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 16:50:38,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 16:50:38,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 16:50:48,683] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-27 16:50:48,968] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:50:49,336] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 16:51:04,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 16:51:05,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 16:51:05,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 16:51:06,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 16:51:06,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 16:51:07,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 16:51:07,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 16:51:08,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 16:51:18,268] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 16:51:18,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 16:51:19,154] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 16:51:34,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 16:51:34,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 16:51:35,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 16:51:35,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 16:51:36,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 16:51:36,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 16:51:37,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 16:51:37,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 16:51:47,944] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 16:55:58,693] INFO : Step 11000/100000 rank6 | Train Loss: 14.4095 | Train Loss Unweighted: 4.7539 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.7591 | Regression_cos_phi: 0.0522 | Regression_energy: 0.4228 | Regression_eta: 0.0709 | Regression_pt: 0.3796 | Regression_sin_phi: 0.0518 | Group Train Loss Unweighted (window mean): physical_pu: 3.8077 | physical_nopu: 4.7444 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5268 | Classification_binary: 3.7134 | Regression_cos_phi: 1.4661 | Regression_energy: 2.7756 | Regression_eta: 0.9498 | Regression_pt: 3.5704 | Regression_sin_phi: 1.9099 | LR: 3.89e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.1995s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1127 | max_loss: 0.0074 | max_backward: 0.1410 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 1.9404 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6580.9 | uss_MB: 5462.7 | child_rss_MB: 10900.1 | allocated_MB: 610.6 | reserved_MB: 27422.0 | inactive_split_MB: 253.4 +rank6 [2026-10-27 16:59:46,314] INFO : Step 12000/100000 rank6 | Train Loss: 16.4725 | Train Loss Unweighted: 5.0863 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.5958 | Regression_cos_phi: 0.0410 | Regression_energy: 0.6842 | Regression_eta: 0.0610 | Regression_pt: 0.6484 | Regression_sin_phi: 0.0394 | Group Train Loss Unweighted (window mean): physical_pu: 3.8013 | physical_nopu: 4.7308 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5618 | Classification_binary: 3.7152 | Regression_cos_phi: 1.5499 | Regression_energy: 3.0194 | Regression_eta: 0.9754 | Regression_pt: 3.8025 | Regression_sin_phi: 1.5579 | LR: 3.87e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.5373s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1292 | max_loss: 0.0037 | max_backward: 0.1951 | max_optimizer_step: 0.2940 | max_tensorboard_logging: 2.1160 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6552.9 | uss_MB: 5462.9 | child_rss_MB: 10800.4 | allocated_MB: 609.5 | reserved_MB: 27422.0 | inactive_split_MB: 347.2 +rank6 [2026-10-27 17:03:40,557] INFO : Step 13000/100000 rank6 | Train Loss: 12.9993 | Train Loss Unweighted: 4.5752 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.7655 | Regression_cos_phi: 0.0483 | Regression_energy: 0.3375 | Regression_eta: 0.0674 | Regression_pt: 0.2918 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7898 | physical_nopu: 4.7063 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3572 | Classification_binary: 3.7003 | Regression_cos_phi: 1.9434 | Regression_energy: 1.5295 | Regression_eta: 0.9436 | Regression_pt: 1.7325 | Regression_sin_phi: 2.0418 | LR: 3.85e-04 | DataLoad Time: 0.0084s | Model Forward Time: 3.0975s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1327 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.9573 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5436.1 | uss_MB: 4383.7 | child_rss_MB: 10499.2 | allocated_MB: 607.2 | reserved_MB: 28584.0 | inactive_split_MB: 340.8 +rank6 [2026-10-27 17:07:23,686] INFO : Step 14000/100000 rank6 | Train Loss: 12.1035 | Train Loss Unweighted: 3.7686 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.0394 | Regression_cos_phi: 0.0492 | Regression_energy: 0.2767 | Regression_eta: 0.1064 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0483 | Group Train Loss Unweighted (window mean): physical_pu: 3.7803 | physical_nopu: 4.6878 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4828 | Classification_binary: 3.0443 | Regression_cos_phi: 1.8359 | Regression_energy: 1.5644 | Regression_eta: 1.7015 | Regression_pt: 1.7432 | Regression_sin_phi: 1.9699 | LR: 3.83e-04 | DataLoad Time: 0.0019s | Model Forward Time: 5.5486s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0034 | max_forward: 0.2002 | max_loss: 0.0036 | max_backward: 0.3668 | max_optimizer_step: 0.2700 | max_tensorboard_logging: 3.9098 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0824 | Memory: rss_MB: 5443.1 | uss_MB: 4393.2 | child_rss_MB: 10497.0 | allocated_MB: 618.5 | reserved_MB: 28584.0 | inactive_split_MB: 893.5 +rank6 [2026-10-27 17:11:07,917] INFO : Step 15000/100000 rank6 | Train Loss: 13.2483 | Train Loss Unweighted: 4.5172 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.6965 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3441 | Regression_eta: 0.0751 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7702 | physical_nopu: 4.6706 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2960 | Classification_binary: 3.7207 | Regression_cos_phi: 1.7248 | Regression_energy: 1.4113 | Regression_eta: 0.8361 | Regression_pt: 1.5736 | Regression_sin_phi: 1.8943 | LR: 3.80e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.3450s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0879 | max_loss: 0.0036 | max_backward: 0.0859 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 2.2540 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0190 | Memory: rss_MB: 5437.1 | uss_MB: 4385.0 | child_rss_MB: 10498.2 | allocated_MB: 605.9 | reserved_MB: 28584.0 | inactive_split_MB: 314.1 +rank6 [2026-10-27 17:14:55,829] INFO : Step 16000/100000 rank6 | Train Loss: 14.0060 | Train Loss Unweighted: 4.5724 | Train Loss Components: Classification: 0.0150 | Classification_binary: 3.5827 | Regression_cos_phi: 0.0469 | Regression_energy: 0.4276 | Regression_eta: 0.0760 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0517 | Group Train Loss Unweighted (window mean): physical_pu: 3.7654 | physical_nopu: 4.6500 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5742 | Classification_binary: 3.6128 | Regression_cos_phi: 2.0569 | Regression_energy: 2.3675 | Regression_eta: 1.0417 | Regression_pt: 2.7761 | Regression_sin_phi: 1.9996 | LR: 3.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.4622s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0984 | max_forward: 0.1749 | max_loss: 0.0039 | max_backward: 0.0976 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 2.1733 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5435.1 | uss_MB: 4384.3 | child_rss_MB: 10495.1 | allocated_MB: 607.6 | reserved_MB: 28978.0 | inactive_split_MB: 330.4 +rank6 [2026-10-27 17:18:33,642] INFO : Step 17000/100000 rank6 | Train Loss: 14.2557 | Train Loss Unweighted: 4.6663 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6292 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4540 | Regression_eta: 0.0715 | Regression_pt: 0.3998 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.7576 | physical_nopu: 4.6430 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3643 | Classification_binary: 3.4140 | Regression_cos_phi: 1.2156 | Regression_energy: 2.9661 | Regression_eta: 0.7451 | Regression_pt: 3.5581 | Regression_sin_phi: 1.5877 | LR: 3.75e-04 | DataLoad Time: 0.0034s | Model Forward Time: 3.9695s | Phase Times: max_pre_step_barrier: 0.5705 | max_batch_to_device: 0.0043 | max_forward: 0.0887 | max_loss: 0.0037 | max_backward: 0.1154 | max_optimizer_step: 0.2542 | max_tensorboard_logging: 2.1801 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5421.0 | uss_MB: 4384.2 | child_rss_MB: 10406.9 | allocated_MB: 609.7 | reserved_MB: 29500.0 | inactive_split_MB: 302.1 +rank6 [2026-10-27 17:22:18,273] INFO : Step 18000/100000 rank6 | Train Loss: 13.2896 | Train Loss Unweighted: 4.5062 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6504 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3607 | Regression_eta: 0.0732 | Regression_pt: 0.3062 | Regression_sin_phi: 0.0530 | Group Train Loss Unweighted (window mean): physical_pu: 3.7500 | physical_nopu: 4.6286 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2790 | Classification_binary: 3.5794 | Regression_cos_phi: 1.6227 | Regression_energy: 1.8712 | Regression_eta: 1.0487 | Regression_pt: 2.0493 | Regression_sin_phi: 2.1562 | LR: 3.72e-04 | DataLoad Time: 0.0020s | Model Forward Time: 10.0051s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0832 | max_loss: 0.0037 | max_backward: 0.0999 | max_optimizer_step: 0.1725 | max_tensorboard_logging: 8.8254 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 5416.7 | uss_MB: 4384.0 | child_rss_MB: 10363.7 | allocated_MB: 608.8 | reserved_MB: 29500.0 | inactive_split_MB: 196.1 +rank6 [2026-10-27 17:25:51,406] INFO : Step 19000/100000 rank6 | Train Loss: 15.2449 | Train Loss Unweighted: 4.8480 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5669 | Regression_cos_phi: 0.0399 | Regression_energy: 0.5792 | Regression_eta: 0.0602 | Regression_pt: 0.5415 | Regression_sin_phi: 0.0443 | Group Train Loss Unweighted (window mean): physical_pu: 3.7399 | physical_nopu: 4.6217 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4331 | Classification_binary: 3.6006 | Regression_cos_phi: 1.4907 | Regression_energy: 2.8087 | Regression_eta: 0.8956 | Regression_pt: 3.4635 | Regression_sin_phi: 2.0122 | LR: 3.69e-04 | DataLoad Time: 0.0027s | Model Forward Time: 3.1640s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0008 | max_forward: 0.0818 | max_loss: 0.0762 | max_backward: 0.1759 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 1.9822 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5410.7 | uss_MB: 4384.1 | child_rss_MB: 10330.8 | allocated_MB: 607.1 | reserved_MB: 29500.0 | inactive_split_MB: 236.2 +rank6 [2026-10-27 17:29:45,026] INFO : Step 20000/100000 rank6 | Train Loss: 13.7275 | Train Loss Unweighted: 4.6495 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.7120 | Regression_cos_phi: 0.0510 | Regression_energy: 0.4061 | Regression_eta: 0.0646 | Regression_pt: 0.3531 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.7387 | physical_nopu: 4.6038 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3894 | Classification_binary: 3.7307 | Regression_cos_phi: 2.0059 | Regression_energy: 2.4336 | Regression_eta: 1.0771 | Regression_pt: 2.8220 | Regression_sin_phi: 2.3446 | LR: 3.66e-04 | DataLoad Time: 0.0027s | Model Forward Time: 5.4931s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0007 | max_forward: 0.0851 | max_loss: 0.0039 | max_backward: 0.2016 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 4.2823 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5411.1 | uss_MB: 4383.8 | child_rss_MB: 10307.1 | allocated_MB: 609.5 | reserved_MB: 29500.0 | inactive_split_MB: 252.8 +rank6 [2026-10-27 17:29:45,969] INFO : Running validation on rank6 +rank6 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 17:31:42,604] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 17:31:43,255] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 17:32:09,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 17:32:10,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 17:32:10,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 17:32:11,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 17:32:11,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 17:32:12,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 17:32:12,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 17:32:13,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 17:32:23,336] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-27 17:32:23,636] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 17:32:24,023] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 17:32:39,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 17:32:39,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 17:32:40,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 17:32:40,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 17:32:41,537] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 17:32:42,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 17:32:42,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 17:32:43,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 17:32:53,105] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-27 17:32:53,432] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 17:32:53,824] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 17:33:08,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 17:33:09,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 17:33:09,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 17:33:10,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 17:33:10,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 17:33:11,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 17:33:11,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 17:33:12,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 17:33:22,015] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 17:37:33,418] INFO : Step 21000/100000 rank6 | Train Loss: 13.1647 | Train Loss Unweighted: 4.3967 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.5106 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3742 | Regression_eta: 0.0796 | Regression_pt: 0.3229 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.7328 | physical_nopu: 4.5933 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4694 | Classification_binary: 3.5350 | Regression_cos_phi: 1.4403 | Regression_energy: 2.3734 | Regression_eta: 1.2054 | Regression_pt: 2.8659 | Regression_sin_phi: 1.5139 | LR: 3.62e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.6676s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0483 | max_forward: 0.1678 | max_loss: 0.0037 | max_backward: 0.1430 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 2.3569 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5516.5 | uss_MB: 4216.2 | child_rss_MB: 10241.6 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 238.6 +rank6 [2026-10-27 17:41:10,019] INFO : Step 22000/100000 rank6 | Train Loss: 15.3802 | Train Loss Unweighted: 4.7914 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.5311 | Regression_cos_phi: 0.0475 | Regression_energy: 0.5593 | Regression_eta: 0.0730 | Regression_pt: 0.5219 | Regression_sin_phi: 0.0434 | Group Train Loss Unweighted (window mean): physical_pu: 3.7290 | physical_nopu: 4.5858 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5076 | Classification_binary: 3.5658 | Regression_cos_phi: 1.6950 | Regression_energy: 3.3566 | Regression_eta: 0.9321 | Regression_pt: 4.1491 | Regression_sin_phi: 1.6020 | LR: 3.59e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.1743s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0905 | max_loss: 0.0038 | max_backward: 0.1361 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 2.0169 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5517.5 | uss_MB: 4498.8 | child_rss_MB: 10247.2 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 321.0 +rank6 [2026-10-27 17:44:47,651] INFO : Step 23000/100000 rank6 | Train Loss: 12.9848 | Train Loss Unweighted: 4.5226 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.7377 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3326 | Regression_eta: 0.0739 | Regression_pt: 0.2617 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.7288 | physical_nopu: 4.5774 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2927 | Classification_binary: 3.8045 | Regression_cos_phi: 2.1780 | Regression_energy: 1.6889 | Regression_eta: 1.1105 | Regression_pt: 1.7218 | Regression_sin_phi: 2.3841 | LR: 3.55e-04 | DataLoad Time: 0.0075s | Model Forward Time: 3.1582s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0994 | max_loss: 0.0039 | max_backward: 0.1244 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.9838 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 5517.5 | uss_MB: 4498.0 | child_rss_MB: 10251.2 | allocated_MB: 608.2 | reserved_MB: 29422.0 | inactive_split_MB: 218.5 +rank6 [2026-10-27 17:48:44,022] INFO : Step 24000/100000 rank6 | Train Loss: 16.3896 | Train Loss Unweighted: 4.8949 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4510 | Regression_cos_phi: 0.0465 | Regression_energy: 0.6613 | Regression_eta: 0.0617 | Regression_pt: 0.6132 | Regression_sin_phi: 0.0453 | Group Train Loss Unweighted (window mean): physical_pu: 3.7170 | physical_nopu: 4.5767 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4379 | Classification_binary: 3.5229 | Regression_cos_phi: 1.8332 | Regression_energy: 2.7892 | Regression_eta: 1.0359 | Regression_pt: 3.2895 | Regression_sin_phi: 2.0474 | LR: 3.51e-04 | DataLoad Time: 0.0028s | Model Forward Time: 3.1022s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0398 | max_forward: 0.1314 | max_loss: 0.0037 | max_backward: 0.0994 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 1.8290 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5518.7 | uss_MB: 4498.4 | child_rss_MB: 10238.8 | allocated_MB: 606.3 | reserved_MB: 29422.0 | inactive_split_MB: 288.6 +rank6 [2026-10-27 17:52:29,098] INFO : Step 25000/100000 rank6 | Train Loss: 13.0062 | Train Loss Unweighted: 4.4048 | Train Loss Components: Classification: 0.0114 | Classification_binary: 3.6384 | Regression_cos_phi: 0.0552 | Regression_energy: 0.3153 | Regression_eta: 0.0680 | Regression_pt: 0.2574 | Regression_sin_phi: 0.0590 | Group Train Loss Unweighted (window mean): physical_pu: 3.7138 | physical_nopu: 4.5644 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2466 | Classification_binary: 3.6863 | Regression_cos_phi: 1.7188 | Regression_energy: 1.4058 | Regression_eta: 1.0558 | Regression_pt: 1.4891 | Regression_sin_phi: 2.1478 | LR: 3.47e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.5990s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0007 | max_forward: 0.1394 | max_loss: 0.0036 | max_backward: 0.2004 | max_optimizer_step: 0.1603 | max_tensorboard_logging: 1.3791 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5517.7 | uss_MB: 4498.3 | child_rss_MB: 10255.6 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 213.9 +rank6 [2026-10-27 17:56:10,619] INFO : Step 26000/100000 rank6 | Train Loss: 13.5469 | Train Loss Unweighted: 4.4523 | Train Loss Components: Classification: 0.0185 | Classification_binary: 3.5246 | Regression_cos_phi: 0.0498 | Regression_energy: 0.3935 | Regression_eta: 0.0732 | Regression_pt: 0.3461 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5624 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4901 | Classification_binary: 3.3944 | Regression_cos_phi: 1.9340 | Regression_energy: 2.0455 | Regression_eta: 1.1329 | Regression_pt: 2.3266 | Regression_sin_phi: 2.0688 | LR: 3.43e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.1211s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0838 | max_forward: 0.1679 | max_loss: 0.0039 | max_backward: 0.0991 | max_optimizer_step: 0.2674 | max_tensorboard_logging: 1.6325 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5515.7 | uss_MB: 4498.4 | child_rss_MB: 10248.9 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 201.9 +rank6 [2026-10-27 17:59:48,627] INFO : Step 27000/100000 rank6 | Train Loss: 17.1780 | Train Loss Unweighted: 5.0531 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.5146 | Regression_cos_phi: 0.0448 | Regression_energy: 0.6955 | Regression_eta: 0.0778 | Regression_pt: 0.6583 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5579 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4429 | Classification_binary: 3.4693 | Regression_cos_phi: 1.5531 | Regression_energy: 3.0365 | Regression_eta: 1.0658 | Regression_pt: 3.6996 | Regression_sin_phi: 1.7111 | LR: 3.39e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4477s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1019 | max_loss: 0.0489 | max_backward: 0.1596 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5507.7 | uss_MB: 4499.5 | child_rss_MB: 10244.8 | allocated_MB: 611.7 | reserved_MB: 30248.0 | inactive_split_MB: 336.3 +rank6 [2026-10-27 18:03:26,754] INFO : Step 28000/100000 rank6 | Train Loss: 12.6721 | Train Loss Unweighted: 4.3436 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5550 | Regression_cos_phi: 0.0488 | Regression_energy: 0.3264 | Regression_eta: 0.0754 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7057 | physical_nopu: 4.5482 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4001 | Classification_binary: 3.4069 | Regression_cos_phi: 1.5368 | Regression_energy: 1.4445 | Regression_eta: 1.1590 | Regression_pt: 1.5529 | Regression_sin_phi: 1.3689 | LR: 3.35e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.0166s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0898 | max_loss: 0.0057 | max_backward: 0.1164 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.8021 | max_tensorboard_flush: 0.0040 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 5489.8 | uss_MB: 4496.9 | child_rss_MB: 10097.2 | allocated_MB: 606.6 | reserved_MB: 30248.0 | inactive_split_MB: 278.7 +rank6 [2026-10-27 18:07:25,425] INFO : Step 29000/100000 rank6 | Train Loss: 14.8403 | Train Loss Unweighted: 4.5753 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.4458 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4901 | Regression_eta: 0.0767 | Regression_pt: 0.4459 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.7068 | physical_nopu: 4.5439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5092 | Classification_binary: 3.5373 | Regression_cos_phi: 2.0592 | Regression_energy: 1.7956 | Regression_eta: 1.2676 | Regression_pt: 1.9562 | Regression_sin_phi: 1.9740 | LR: 3.30e-04 | DataLoad Time: 0.0026s | Model Forward Time: 2.3378s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1464 | max_loss: 0.0037 | max_backward: 0.0935 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 1.1535 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5489.8 | uss_MB: 4498.4 | child_rss_MB: 10094.9 | allocated_MB: 607.4 | reserved_MB: 30248.0 | inactive_split_MB: 313.3 +rank6 [2026-10-27 18:11:12,494] INFO : Step 30000/100000 rank6 | Train Loss: 13.2916 | Train Loss Unweighted: 4.4954 | Train Loss Components: Classification: 0.0126 | Classification_binary: 3.5989 | Regression_cos_phi: 0.0482 | Regression_energy: 0.3865 | Regression_eta: 0.0707 | Regression_pt: 0.3291 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7005 | physical_nopu: 4.5378 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3406 | Classification_binary: 3.6783 | Regression_cos_phi: 1.8298 | Regression_energy: 1.6100 | Regression_eta: 1.0729 | Regression_pt: 1.7733 | Regression_sin_phi: 2.1091 | LR: 3.26e-04 | DataLoad Time: 0.0013s | Model Forward Time: 2.7234s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0805 | max_loss: 0.0038 | max_backward: 0.0926 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 1.5279 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 5479.8 | uss_MB: 4498.3 | child_rss_MB: 10016.6 | allocated_MB: 608.2 | reserved_MB: 30760.0 | inactive_split_MB: 232.9 +rank6 [2026-10-27 18:11:13,431] INFO : Running validation on rank6 +rank6 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 18:11:13,432] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 18:11:13,432] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 18:13:07,587] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 18:13:08,048] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 18:13:34,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 18:13:35,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 18:13:35,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 18:13:36,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 18:13:36,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 18:13:37,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 18:13:37,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 18:13:38,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 18:13:48,079] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-27 18:13:48,509] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 18:13:49,355] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 18:14:04,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 18:14:05,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 18:14:05,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 18:14:06,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 18:14:07,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 18:14:07,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 18:14:08,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 18:14:08,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 18:14:18,579] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 18:14:18,746] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 18:14:19,219] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 18:14:34,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 18:14:35,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 18:14:35,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 18:14:36,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 18:14:36,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 18:14:37,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 18:14:37,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 18:14:38,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 18:14:48,013] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 18:19:00,303] INFO : Step 31000/100000 rank6 | Train Loss: 12.7750 | Train Loss Unweighted: 4.3889 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5406 | Regression_cos_phi: 0.0465 | Regression_energy: 0.3622 | Regression_eta: 0.0700 | Regression_pt: 0.3101 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6964 | physical_nopu: 4.5360 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3495 | Classification_binary: 3.5018 | Regression_cos_phi: 1.5048 | Regression_energy: 1.7076 | Regression_eta: 0.8962 | Regression_pt: 1.8155 | Regression_sin_phi: 1.5116 | LR: 3.21e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.3236s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.2346 | max_loss: 0.0037 | max_backward: 0.1255 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 1.8384 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1395 | Memory: rss_MB: 6773.5 | uss_MB: 5791.8 | child_rss_MB: 10011.8 | allocated_MB: 609.3 | reserved_MB: 29142.0 | inactive_split_MB: 278.5 +rank6 [2026-10-27 18:22:44,788] INFO : Step 32000/100000 rank6 | Train Loss: 14.3617 | Train Loss Unweighted: 4.6225 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4934 | Regression_cos_phi: 0.0448 | Regression_energy: 0.5043 | Regression_eta: 0.0718 | Regression_pt: 0.4507 | Regression_sin_phi: 0.0414 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5314 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4835 | Classification_binary: 3.4730 | Regression_cos_phi: 1.8672 | Regression_energy: 2.9655 | Regression_eta: 1.7201 | Regression_pt: 3.2541 | Regression_sin_phi: 2.3189 | LR: 3.16e-04 | DataLoad Time: 0.0100s | Model Forward Time: 2.5627s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1373 | max_loss: 0.0041 | max_backward: 0.2040 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 1.2065 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5556.1 | uss_MB: 4574.8 | child_rss_MB: 10008.9 | allocated_MB: 606.1 | reserved_MB: 29142.0 | inactive_split_MB: 511.2 +rank6 [2026-10-27 18:26:33,026] INFO : Step 33000/100000 rank6 | Train Loss: 12.0000 | Train Loss Unweighted: 3.7115 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9674 | Regression_cos_phi: 0.0474 | Regression_energy: 0.2863 | Regression_eta: 0.1056 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4696 | Classification_binary: 3.0328 | Regression_cos_phi: 1.8452 | Regression_energy: 1.6454 | Regression_eta: 1.6101 | Regression_pt: 1.9454 | Regression_sin_phi: 1.8993 | LR: 3.12e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.5988s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0030 | max_forward: 0.1619 | max_loss: 0.0119 | max_backward: 0.2845 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.1927 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5557.1 | uss_MB: 4556.3 | child_rss_MB: 10013.2 | allocated_MB: 623.7 | reserved_MB: 29572.0 | inactive_split_MB: 1441.5 +rank6 [2026-10-27 18:30:15,971] INFO : Step 34000/100000 rank6 | Train Loss: 15.0627 | Train Loss Unweighted: 4.6899 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4670 | Regression_cos_phi: 0.0454 | Regression_energy: 0.5438 | Regression_eta: 0.0626 | Regression_pt: 0.5082 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6889 | physical_nopu: 4.5215 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5329 | Classification_binary: 3.3355 | Regression_cos_phi: 1.3688 | Regression_energy: 3.6365 | Regression_eta: 0.9430 | Regression_pt: 4.8819 | Regression_sin_phi: 1.5711 | LR: 3.07e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.3754s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0007 | max_forward: 0.1346 | max_loss: 0.0037 | max_backward: 0.1938 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1506 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5552.6 | uss_MB: 4573.6 | child_rss_MB: 9984.9 | allocated_MB: 609.6 | reserved_MB: 29572.0 | inactive_split_MB: 230.4 +rank6 [2026-10-27 18:33:55,272] INFO : Step 35000/100000 rank6 | Train Loss: 12.2019 | Train Loss Unweighted: 4.3540 | Train Loss Components: Classification: 0.0089 | Classification_binary: 3.6426 | Regression_cos_phi: 0.0532 | Regression_energy: 0.2965 | Regression_eta: 0.0664 | Regression_pt: 0.2370 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6854 | physical_nopu: 4.5184 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3156 | Classification_binary: 3.6738 | Regression_cos_phi: 2.0291 | Regression_energy: 1.6012 | Regression_eta: 1.0097 | Regression_pt: 1.7722 | Regression_sin_phi: 2.1095 | LR: 3.02e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.6472s | Phase Times: max_pre_step_barrier: 0.0511 | max_batch_to_device: 0.0857 | max_forward: 0.1832 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 1.2656 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 5548.6 | uss_MB: 4574.4 | child_rss_MB: 9918.7 | allocated_MB: 608.2 | reserved_MB: 29572.0 | inactive_split_MB: 245.8 +rank6 [2026-10-27 18:37:33,850] INFO : Step 36000/100000 rank6 | Train Loss: 12.7130 | Train Loss Unweighted: 4.3877 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.5479 | Regression_cos_phi: 0.0433 | Regression_energy: 0.3521 | Regression_eta: 0.0636 | Regression_pt: 0.3167 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.6860 | physical_nopu: 4.5137 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4126 | Classification_binary: 3.6741 | Regression_cos_phi: 1.4916 | Regression_energy: 1.7455 | Regression_eta: 0.9127 | Regression_pt: 2.1769 | Regression_sin_phi: 1.7230 | LR: 2.97e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.5813s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0007 | max_forward: 0.1685 | max_loss: 0.0137 | max_backward: 0.0851 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.2906 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 5532.6 | uss_MB: 4487.8 | child_rss_MB: 9818.0 | allocated_MB: 609.0 | reserved_MB: 29572.0 | inactive_split_MB: 268.4 +rank6 [2026-10-27 18:41:12,499] INFO : Step 37000/100000 rank6 | Train Loss: 16.1273 | Train Loss Unweighted: 4.7522 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4049 | Regression_cos_phi: 0.0495 | Regression_energy: 0.5975 | Regression_eta: 0.0881 | Regression_pt: 0.5494 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6829 | physical_nopu: 4.5107 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5503 | Classification_binary: 3.3794 | Regression_cos_phi: 2.1046 | Regression_energy: 2.3635 | Regression_eta: 1.3944 | Regression_pt: 2.9508 | Regression_sin_phi: 2.1970 | LR: 2.91e-04 | DataLoad Time: 0.0029s | Model Forward Time: 2.3163s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0884 | max_loss: 0.0036 | max_backward: 0.0963 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 1.0635 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0656 | Memory: rss_MB: 5529.2 | uss_MB: 4574.6 | child_rss_MB: 9790.1 | allocated_MB: 605.6 | reserved_MB: 29572.0 | inactive_split_MB: 233.2 +rank6 [2026-10-27 18:44:44,282] INFO : Step 38000/100000 rank6 | Train Loss: 12.5142 | Train Loss Unweighted: 4.4218 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.6290 | Regression_cos_phi: 0.0475 | Regression_energy: 0.3376 | Regression_eta: 0.0660 | Regression_pt: 0.2827 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6875 | physical_nopu: 4.5136 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4037 | Classification_binary: 3.7995 | Regression_cos_phi: 1.6831 | Regression_energy: 1.5992 | Regression_eta: 1.0433 | Regression_pt: 1.6758 | Regression_sin_phi: 1.9450 | LR: 2.86e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.4324s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.0807 | max_loss: 0.0039 | max_backward: 0.1583 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.2386 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5519.2 | uss_MB: 4574.3 | child_rss_MB: 9713.9 | allocated_MB: 607.7 | reserved_MB: 29572.0 | inactive_split_MB: 203.1 +rank6 [2026-10-27 18:48:30,202] INFO : Step 39000/100000 rank6 | Train Loss: 13.9834 | Train Loss Unweighted: 4.4509 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4391 | Regression_cos_phi: 0.0477 | Regression_energy: 0.4363 | Regression_eta: 0.0792 | Regression_pt: 0.3839 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.6802 | physical_nopu: 4.5058 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4268 | Classification_binary: 3.4321 | Regression_cos_phi: 1.5379 | Regression_energy: 3.1110 | Regression_eta: 1.2749 | Regression_pt: 3.4968 | Regression_sin_phi: 1.9819 | LR: 2.81e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.8720s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.0950 | max_loss: 0.0039 | max_backward: 0.1102 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 2.6970 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5506.6 | uss_MB: 4574.6 | child_rss_MB: 9623.2 | allocated_MB: 611.4 | reserved_MB: 29682.0 | inactive_split_MB: 276.6 +rank6 [2026-10-27 18:52:07,824] INFO : Step 40000/100000 rank6 | Train Loss: 15.4467 | Train Loss Unweighted: 4.8241 | Train Loss Components: Classification: 0.0121 | Classification_binary: 3.5538 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5661 | Regression_eta: 0.0744 | Regression_pt: 0.5256 | Regression_sin_phi: 0.0462 | Group Train Loss Unweighted (window mean): physical_pu: 3.6754 | physical_nopu: 4.5045 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4124 | Classification_binary: 3.4058 | Regression_cos_phi: 2.0711 | Regression_energy: 2.8241 | Regression_eta: 1.4868 | Regression_pt: 4.0026 | Regression_sin_phi: 1.6030 | LR: 2.76e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.4738s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.1251 | max_loss: 0.0037 | max_backward: 0.0913 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2799 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5491.6 | uss_MB: 4574.5 | child_rss_MB: 9510.6 | allocated_MB: 607.6 | reserved_MB: 29682.0 | inactive_split_MB: 251.4 +rank6 [2026-10-27 18:52:08,683] INFO : Running validation on rank6 +rank6 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 18:54:11,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 18:54:12,296] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 18:54:37,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 18:54:37,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 18:54:38,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 18:54:38,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 18:54:39,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 18:54:39,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 18:54:40,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 18:54:41,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 18:54:51,015] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-27 18:54:51,320] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 18:54:51,717] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 18:55:06,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 18:55:07,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 18:55:07,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 18:55:08,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 18:55:08,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 18:55:09,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 18:55:09,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 18:55:10,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 18:55:20,458] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 18:55:21,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 18:55:21,564] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 18:55:36,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 18:55:37,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 18:55:37,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 18:55:38,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 18:55:38,903] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 18:55:39,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 18:55:39,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 18:55:40,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 18:55:50,390] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 19:00:08,492] INFO : Step 41000/100000 rank6 | Train Loss: 13.2101 | Train Loss Unweighted: 4.4466 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5620 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3777 | Regression_eta: 0.0720 | Regression_pt: 0.3238 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6774 | physical_nopu: 4.4925 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5110 | Classification_binary: 3.4040 | Regression_cos_phi: 1.5921 | Regression_energy: 1.7919 | Regression_eta: 0.7327 | Regression_pt: 2.1589 | Regression_sin_phi: 2.1057 | LR: 2.70e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.5584s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0904 | max_forward: 0.2227 | max_loss: 0.0038 | max_backward: 0.1239 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 2.1580 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6748.4 | uss_MB: 5831.9 | child_rss_MB: 9514.0 | allocated_MB: 608.3 | reserved_MB: 27230.0 | inactive_split_MB: 387.2 +rank6 [2026-10-27 19:03:45,853] INFO : Step 42000/100000 rank6 | Train Loss: 13.7061 | Train Loss Unweighted: 4.4760 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4108 | Regression_cos_phi: 0.0459 | Regression_energy: 0.4702 | Regression_eta: 0.0614 | Regression_pt: 0.4341 | Regression_sin_phi: 0.0374 | Group Train Loss Unweighted (window mean): physical_pu: 3.6711 | physical_nopu: 4.4967 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5087 | Classification_binary: 3.5700 | Regression_cos_phi: 1.2345 | Regression_energy: 2.3082 | Regression_eta: 0.9902 | Regression_pt: 2.7662 | Regression_sin_phi: 2.0709 | LR: 2.65e-04 | DataLoad Time: 0.0073s | Model Forward Time: 2.9856s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0008 | max_forward: 0.1920 | max_loss: 0.0038 | max_backward: 0.1427 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 1.6213 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6739.4 | uss_MB: 5628.8 | child_rss_MB: 9436.4 | allocated_MB: 609.4 | reserved_MB: 27632.0 | inactive_split_MB: 509.5 +rank6 [2026-10-27 19:07:23,507] INFO : Step 43000/100000 rank6 | Train Loss: 11.9489 | Train Loss Unweighted: 3.6938 | Train Loss Components: Classification: 0.0162 | Classification_binary: 2.9682 | Regression_cos_phi: 0.0472 | Regression_energy: 0.2764 | Regression_eta: 0.1089 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6702 | physical_nopu: 4.4944 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4740 | Classification_binary: 2.9383 | Regression_cos_phi: 1.7525 | Regression_energy: 1.4788 | Regression_eta: 1.5848 | Regression_pt: 1.6477 | Regression_sin_phi: 1.8817 | LR: 2.59e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.8332s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0032 | max_forward: 0.2195 | max_loss: 0.0114 | max_backward: 0.4513 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 2.0889 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0865 | Memory: rss_MB: 5560.5 | uss_MB: 3524.0 | child_rss_MB: 9348.2 | allocated_MB: 619.9 | reserved_MB: 27918.0 | inactive_split_MB: 1305.5 +rank6 [2026-10-27 19:11:14,852] INFO : Step 44000/100000 rank6 | Train Loss: 15.8790 | Train Loss Unweighted: 4.7893 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4610 | Regression_cos_phi: 0.0438 | Regression_energy: 0.5944 | Regression_eta: 0.0747 | Regression_pt: 0.5488 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6681 | physical_nopu: 4.4856 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5539 | Classification_binary: 3.2488 | Regression_cos_phi: 1.3145 | Regression_energy: 2.6044 | Regression_eta: 0.8274 | Regression_pt: 3.3289 | Regression_sin_phi: 1.5501 | LR: 2.54e-04 | DataLoad Time: 0.0028s | Model Forward Time: 3.1646s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0980 | max_loss: 0.0531 | max_backward: 0.1371 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.9403 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5553.5 | uss_MB: 4666.8 | child_rss_MB: 9287.7 | allocated_MB: 606.4 | reserved_MB: 31154.0 | inactive_split_MB: 295.0 +rank6 [2026-10-27 19:15:01,382] INFO : Step 45000/100000 rank6 | Train Loss: 13.8786 | Train Loss Unweighted: 4.5241 | Train Loss Components: Classification: 0.0100 | Classification_binary: 3.5822 | Regression_cos_phi: 0.0536 | Regression_energy: 0.4068 | Regression_eta: 0.0847 | Regression_pt: 0.3344 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 3.6667 | physical_nopu: 4.4836 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3379 | Classification_binary: 3.6132 | Regression_cos_phi: 2.1455 | Regression_energy: 1.7727 | Regression_eta: 1.0877 | Regression_pt: 1.8747 | Regression_sin_phi: 2.1586 | LR: 2.48e-04 | DataLoad Time: 0.0010s | Model Forward Time: 4.6047s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0009 | max_forward: 0.0951 | max_loss: 0.0815 | max_backward: 0.1837 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 3.3466 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 5551.4 | uss_MB: 4666.0 | child_rss_MB: 9284.1 | allocated_MB: 608.8 | reserved_MB: 31154.0 | inactive_split_MB: 289.8 +rank6 [2026-10-27 19:18:42,715] INFO : Step 46000/100000 rank6 | Train Loss: 13.1587 | Train Loss Unweighted: 4.3779 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.4817 | Regression_cos_phi: 0.0459 | Regression_energy: 0.3805 | Regression_eta: 0.0668 | Regression_pt: 0.3375 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.6620 | physical_nopu: 4.4831 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4545 | Classification_binary: 3.6080 | Regression_cos_phi: 1.6095 | Regression_energy: 1.9743 | Regression_eta: 0.9796 | Regression_pt: 2.3391 | Regression_sin_phi: 2.4884 | LR: 2.43e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.3332s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0924 | max_forward: 0.1910 | max_loss: 0.0039 | max_backward: 0.1028 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 2.0151 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5545.4 | uss_MB: 4666.2 | child_rss_MB: 9248.1 | allocated_MB: 609.2 | reserved_MB: 31154.0 | inactive_split_MB: 277.4 +rank6 [2026-10-27 19:22:31,522] INFO : Step 47000/100000 rank6 | Train Loss: 15.2118 | Train Loss Unweighted: 4.6831 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3624 | Regression_cos_phi: 0.0409 | Regression_energy: 0.5987 | Regression_eta: 0.0659 | Regression_pt: 0.5598 | Regression_sin_phi: 0.0405 | Group Train Loss Unweighted (window mean): physical_pu: 3.6609 | physical_nopu: 4.4795 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3674 | Classification_binary: 3.4259 | Regression_cos_phi: 1.8867 | Regression_energy: 2.0679 | Regression_eta: 0.9755 | Regression_pt: 2.5230 | Regression_sin_phi: 1.8951 | LR: 2.37e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.2745s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0777 | max_forward: 0.1864 | max_loss: 0.0077 | max_backward: 0.1105 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 1.8392 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5533.2 | uss_MB: 4665.8 | child_rss_MB: 9129.7 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 305.0 +rank6 [2026-10-27 19:26:21,057] INFO : Step 48000/100000 rank6 | Train Loss: 12.0455 | Train Loss Unweighted: 4.3014 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.6150 | Regression_cos_phi: 0.0526 | Regression_energy: 0.2805 | Regression_eta: 0.0652 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6617 | physical_nopu: 4.4726 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2411 | Classification_binary: 3.7782 | Regression_cos_phi: 1.8883 | Regression_energy: 1.7046 | Regression_eta: 1.3523 | Regression_pt: 1.7273 | Regression_sin_phi: 2.2260 | LR: 2.31e-04 | DataLoad Time: 0.0009s | Model Forward Time: 3.3658s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0531 | max_forward: 0.1607 | max_loss: 0.0044 | max_backward: 0.0997 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 2.0345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5534.2 | uss_MB: 4664.3 | child_rss_MB: 9129.2 | allocated_MB: 606.8 | reserved_MB: 31154.0 | inactive_split_MB: 262.3 +rank6 [2026-10-27 19:30:06,213] INFO : Step 49000/100000 rank6 | Train Loss: 13.3973 | Train Loss Unweighted: 4.4035 | Train Loss Components: Classification: 0.0168 | Classification_binary: 3.5033 | Regression_cos_phi: 0.0506 | Regression_energy: 0.3797 | Regression_eta: 0.0752 | Regression_pt: 0.3299 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6566 | physical_nopu: 4.4735 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5428 | Classification_binary: 3.6031 | Regression_cos_phi: 1.8951 | Regression_energy: 1.9787 | Regression_eta: 1.1477 | Regression_pt: 2.3384 | Regression_sin_phi: 1.8277 | LR: 2.26e-04 | DataLoad Time: 0.0022s | Model Forward Time: 4.5484s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0220 | max_forward: 0.1075 | max_loss: 0.0037 | max_backward: 0.1020 | max_optimizer_step: 0.2722 | max_tensorboard_logging: 3.2606 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5532.9 | uss_MB: 4389.0 | child_rss_MB: 9133.5 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 312.3 +rank6 [2026-10-27 19:33:47,826] INFO : Step 50000/100000 rank6 | Train Loss: 14.2027 | Train Loss Unweighted: 4.5299 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4272 | Regression_cos_phi: 0.0432 | Regression_energy: 0.4877 | Regression_eta: 0.0670 | Regression_pt: 0.4429 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4710 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5668 | Classification_binary: 3.3237 | Regression_cos_phi: 1.7133 | Regression_energy: 2.3205 | Regression_eta: 1.1517 | Regression_pt: 2.9359 | Regression_sin_phi: 2.2860 | LR: 2.20e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.0502s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0964 | max_loss: 0.0590 | max_backward: 0.1520 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.8031 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5519.9 | uss_MB: 4665.8 | child_rss_MB: 9015.1 | allocated_MB: 607.2 | reserved_MB: 31154.0 | inactive_split_MB: 255.6 +rank6 [2026-10-27 19:33:48,728] INFO : Running validation on rank6 +rank6 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 19:35:46,022] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 19:35:46,497] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 19:36:12,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 19:36:12,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 19:36:13,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 19:36:13,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 19:36:14,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 19:36:14,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 19:36:15,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 19:36:15,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 19:36:25,735] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-27 19:36:25,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 19:36:26,397] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 19:36:41,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 19:36:41,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 19:36:42,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 19:36:42,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 19:36:43,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 19:36:44,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 19:36:44,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 19:36:45,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 19:36:55,071] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 19:36:55,547] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 19:36:55,958] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 19:37:11,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 19:37:11,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 19:37:12,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 19:37:12,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 19:37:13,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 19:37:13,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 19:37:14,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 19:37:14,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 19:37:24,812] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 19:41:35,792] INFO : Step 51000/100000 rank6 | Train Loss: 13.0997 | Train Loss Unweighted: 4.3733 | Train Loss Components: Classification: 0.0129 | Classification_binary: 3.5282 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3598 | Regression_eta: 0.0785 | Regression_pt: 0.2920 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6545 | physical_nopu: 4.4631 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4924 | Classification_binary: 3.3934 | Regression_cos_phi: 1.7359 | Regression_energy: 1.6787 | Regression_eta: 0.7643 | Regression_pt: 1.8074 | Regression_sin_phi: 2.1703 | LR: 2.14e-04 | DataLoad Time: 0.0114s | Model Forward Time: 3.1040s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1313 | max_loss: 0.0271 | max_backward: 0.1689 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.7590 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6858.3 | uss_MB: 6017.9 | child_rss_MB: 8958.7 | allocated_MB: 607.6 | reserved_MB: 29536.0 | inactive_split_MB: 388.7 +rank6 [2026-10-27 19:45:15,993] INFO : Step 52000/100000 rank6 | Train Loss: 11.7496 | Train Loss Unweighted: 3.6574 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9475 | Regression_cos_phi: 0.0469 | Regression_energy: 0.2717 | Regression_eta: 0.1056 | Regression_pt: 0.2231 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6567 | physical_nopu: 4.4685 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4743 | Classification_binary: 3.0457 | Regression_cos_phi: 1.7817 | Regression_energy: 1.4727 | Regression_eta: 1.6368 | Regression_pt: 1.5918 | Regression_sin_phi: 1.9820 | LR: 2.09e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.6141s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.1059 | max_forward: 0.2956 | max_loss: 0.0038 | max_backward: 0.4187 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 1.8177 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6858.3 | uss_MB: 6026.1 | child_rss_MB: 8942.2 | allocated_MB: 619.9 | reserved_MB: 29990.0 | inactive_split_MB: 1078.6 +rank6 [2026-10-27 19:48:51,528] INFO : Step 53000/100000 rank6 | Train Loss: 12.5294 | Train Loss Unweighted: 4.3083 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5302 | Regression_cos_phi: 0.0494 | Regression_energy: 0.3242 | Regression_eta: 0.0722 | Regression_pt: 0.2705 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6530 | physical_nopu: 4.4629 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3863 | Classification_binary: 3.4083 | Regression_cos_phi: 1.4556 | Regression_energy: 1.5579 | Regression_eta: 1.2641 | Regression_pt: 1.7389 | Regression_sin_phi: 1.6676 | LR: 2.03e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.5886s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.1740 | max_loss: 0.0036 | max_backward: 0.1135 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 1.3124 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 5606.5 | uss_MB: 4766.5 | child_rss_MB: 8943.1 | allocated_MB: 608.5 | reserved_MB: 29990.0 | inactive_split_MB: 246.4 +rank6 [2026-10-27 19:52:28,508] INFO : Step 54000/100000 rank6 | Train Loss: 13.6142 | Train Loss Unweighted: 4.3355 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.3742 | Regression_cos_phi: 0.0499 | Regression_energy: 0.4083 | Regression_eta: 0.0736 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4652 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5078 | Classification_binary: 3.2011 | Regression_cos_phi: 2.0421 | Regression_energy: 2.3748 | Regression_eta: 0.9020 | Regression_pt: 2.8236 | Regression_sin_phi: 1.9943 | LR: 1.97e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.4442s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1029 | max_loss: 0.0156 | max_backward: 0.1263 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2459 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 5600.5 | uss_MB: 4766.6 | child_rss_MB: 8886.2 | allocated_MB: 608.0 | reserved_MB: 29990.0 | inactive_split_MB: 249.1 +rank6 [2026-10-27 19:56:02,481] INFO : Step 55000/100000 rank6 | Train Loss: 14.4463 | Train Loss Unweighted: 4.6356 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5250 | Regression_cos_phi: 0.0497 | Regression_energy: 0.4959 | Regression_eta: 0.0660 | Regression_pt: 0.4403 | Regression_sin_phi: 0.0455 | Group Train Loss Unweighted (window mean): physical_pu: 3.6517 | physical_nopu: 4.4554 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3268 | Classification_binary: 3.6656 | Regression_cos_phi: 2.0073 | Regression_energy: 1.9146 | Regression_eta: 1.4056 | Regression_pt: 2.1670 | Regression_sin_phi: 2.1947 | LR: 1.92e-04 | DataLoad Time: 0.0028s | Model Forward Time: 2.9797s | Phase Times: max_pre_step_barrier: 0.0108 | max_batch_to_device: 0.0008 | max_forward: 0.0923 | max_loss: 0.0219 | max_backward: 0.1306 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 1.7796 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5599.8 | uss_MB: 4765.5 | child_rss_MB: 8883.4 | allocated_MB: 610.0 | reserved_MB: 29990.0 | inactive_split_MB: 274.0 +rank6 [2026-10-27 19:59:40,797] INFO : Step 56000/100000 rank6 | Train Loss: 12.4091 | Train Loss Unweighted: 4.2572 | Train Loss Components: Classification: 0.0135 | Classification_binary: 3.4523 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3421 | Regression_eta: 0.0664 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6493 | physical_nopu: 4.4547 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4046 | Classification_binary: 3.3999 | Regression_cos_phi: 1.6871 | Regression_energy: 2.0450 | Regression_eta: 1.0622 | Regression_pt: 2.3469 | Regression_sin_phi: 1.7534 | LR: 1.86e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.0679s | Phase Times: max_pre_step_barrier: 0.3290 | max_batch_to_device: 0.0902 | max_forward: 0.1771 | max_loss: 0.0038 | max_backward: 0.1058 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 1.4273 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5599.8 | uss_MB: 4766.9 | child_rss_MB: 8880.9 | allocated_MB: 608.1 | reserved_MB: 29990.0 | inactive_split_MB: 253.9 +rank6 [2026-10-27 20:03:29,496] INFO : Step 57000/100000 rank6 | Train Loss: 14.9087 | Train Loss Unweighted: 4.5957 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4371 | Regression_cos_phi: 0.0471 | Regression_energy: 0.5028 | Regression_eta: 0.0770 | Regression_pt: 0.4658 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6475 | physical_nopu: 4.4509 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3787 | Classification_binary: 3.6596 | Regression_cos_phi: 1.4049 | Regression_energy: 2.2143 | Regression_eta: 0.7859 | Regression_pt: 2.6682 | Regression_sin_phi: 1.6551 | LR: 1.81e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.4960s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1036 | max_loss: 0.0876 | max_backward: 0.2130 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 1.1951 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5597.8 | uss_MB: 4497.3 | child_rss_MB: 8866.5 | allocated_MB: 608.1 | reserved_MB: 30358.0 | inactive_split_MB: 283.9 +rank6 [2026-10-27 20:07:08,735] INFO : Step 58000/100000 rank6 | Train Loss: 12.4468 | Train Loss Unweighted: 4.2826 | Train Loss Components: Classification: 0.0105 | Classification_binary: 3.5299 | Regression_cos_phi: 0.0545 | Regression_energy: 0.3106 | Regression_eta: 0.0620 | Regression_pt: 0.2643 | Regression_sin_phi: 0.0509 | Group Train Loss Unweighted (window mean): physical_pu: 3.6440 | physical_nopu: 4.4492 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3483 | Classification_binary: 3.5443 | Regression_cos_phi: 1.9439 | Regression_energy: 1.6562 | Regression_eta: 0.9298 | Regression_pt: 1.8980 | Regression_sin_phi: 2.0745 | LR: 1.75e-04 | DataLoad Time: 0.0027s | Model Forward Time: 2.5423s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1868 | max_loss: 0.0037 | max_backward: 0.1111 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2860 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5587.6 | uss_MB: 3611.2 | child_rss_MB: 8803.1 | allocated_MB: 608.2 | reserved_MB: 30358.0 | inactive_split_MB: 337.0 +rank6 [2026-10-27 20:10:52,442] INFO : Step 59000/100000 rank6 | Train Loss: 13.1870 | Train Loss Unweighted: 4.3659 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4760 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3771 | Regression_eta: 0.0773 | Regression_pt: 0.3247 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4440 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4438 | Classification_binary: 3.4124 | Regression_cos_phi: 1.5655 | Regression_energy: 1.7083 | Regression_eta: 0.9663 | Regression_pt: 2.0559 | Regression_sin_phi: 1.6644 | LR: 1.70e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.4661s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1005 | max_loss: 0.0036 | max_backward: 0.1062 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 1.2390 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 5582.6 | uss_MB: 4766.7 | child_rss_MB: 8755.9 | allocated_MB: 610.1 | reserved_MB: 31234.0 | inactive_split_MB: 273.9 +rank6 [2026-10-27 20:14:34,818] INFO : Step 60000/100000 rank6 | Train Loss: 15.4000 | Train Loss Unweighted: 4.6905 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.3905 | Regression_cos_phi: 0.0456 | Regression_energy: 0.5805 | Regression_eta: 0.0703 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6406 | physical_nopu: 4.4523 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4262 | Classification_binary: 3.3537 | Regression_cos_phi: 1.8510 | Regression_energy: 3.7937 | Regression_eta: 1.2927 | Regression_pt: 5.0957 | Regression_sin_phi: 2.3289 | LR: 1.64e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.5443s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0757 | max_forward: 0.1624 | max_loss: 0.0040 | max_backward: 0.1315 | max_optimizer_step: 0.2155 | max_tensorboard_logging: 1.2531 | max_tensorboard_flush: 0.0020 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5582.8 | uss_MB: 4766.6 | child_rss_MB: 8760.5 | allocated_MB: 611.6 | reserved_MB: 31234.0 | inactive_split_MB: 318.4 +rank6 [2026-10-27 20:14:35,735] INFO : Running validation on rank6 +rank6 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 20:14:35,736] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 20:16:30,818] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 20:16:31,258] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 20:16:55,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 20:16:56,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 20:16:56,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 20:16:57,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 20:16:57,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 20:16:58,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 20:16:58,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 20:16:59,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 20:17:09,346] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-27 20:17:09,354] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 20:17:09,706] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 20:17:24,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 20:17:25,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 20:17:25,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 20:17:26,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 20:17:26,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 20:17:27,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 20:17:27,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 20:17:28,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 20:17:38,327] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 20:17:39,021] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 20:17:39,450] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 20:17:55,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 20:17:55,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 20:17:56,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 20:17:56,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 20:17:57,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 20:17:57,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 20:17:58,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 20:17:58,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 20:18:08,639] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 20:22:28,768] INFO : Step 61000/100000 rank6 | Train Loss: 12.1474 | Train Loss Unweighted: 4.3289 | Train Loss Components: Classification: 0.0110 | Classification_binary: 3.6236 | Regression_cos_phi: 0.0523 | Regression_energy: 0.2909 | Regression_eta: 0.0654 | Regression_pt: 0.2368 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.6358 | physical_nopu: 4.4439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3191 | Classification_binary: 3.5405 | Regression_cos_phi: 2.4159 | Regression_energy: 1.5524 | Regression_eta: 1.0511 | Regression_pt: 1.5845 | Regression_sin_phi: 2.1405 | LR: 1.59e-04 | DataLoad Time: 0.0074s | Model Forward Time: 3.4732s | Phase Times: max_pre_step_barrier: 0.0115 | max_batch_to_device: 0.0009 | max_forward: 0.2092 | max_loss: 0.0037 | max_backward: 0.1501 | max_optimizer_step: 0.2965 | max_tensorboard_logging: 1.8725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6817.2 | uss_MB: 6009.9 | child_rss_MB: 8755.0 | allocated_MB: 607.8 | reserved_MB: 28822.0 | inactive_split_MB: 222.9 +rank6 [2026-10-27 20:26:14,158] INFO : Step 62000/100000 rank6 | Train Loss: 11.7977 | Train Loss Unweighted: 3.6294 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9084 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2778 | Regression_eta: 0.1039 | Regression_pt: 0.2286 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6417 | physical_nopu: 4.4459 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4560 | Classification_binary: 2.9604 | Regression_cos_phi: 1.8393 | Regression_energy: 1.6155 | Regression_eta: 1.5622 | Regression_pt: 1.8266 | Regression_sin_phi: 1.8862 | LR: 1.54e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.7425s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0033 | max_forward: 0.2219 | max_loss: 0.0037 | max_backward: 0.2965 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1292 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1278 | Memory: rss_MB: 6812.4 | uss_MB: 6017.7 | child_rss_MB: 8760.3 | allocated_MB: 618.8 | reserved_MB: 28822.0 | inactive_split_MB: 888.0 +rank6 [2026-10-27 20:30:05,877] INFO : Step 63000/100000 rank6 | Train Loss: 12.5071 | Train Loss Unweighted: 4.3613 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5997 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3200 | Regression_eta: 0.0719 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6399 | physical_nopu: 4.4447 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4441 | Classification_binary: 3.6732 | Regression_cos_phi: 1.8146 | Regression_energy: 1.7409 | Regression_eta: 1.1809 | Regression_pt: 1.7179 | Regression_sin_phi: 1.9761 | LR: 1.49e-04 | DataLoad Time: 0.0023s | Model Forward Time: 11.0658s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1169 | max_loss: 0.0157 | max_backward: 0.2882 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 9.0944 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6806.2 | uss_MB: 4694.0 | child_rss_MB: 8747.4 | allocated_MB: 610.4 | reserved_MB: 28822.0 | inactive_split_MB: 261.6 +rank6 [2026-10-27 20:33:53,443] INFO : Step 64000/100000 rank6 | Train Loss: 14.0388 | Train Loss Unweighted: 4.5035 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.4200 | Regression_cos_phi: 0.0430 | Regression_energy: 0.4752 | Regression_eta: 0.0663 | Regression_pt: 0.4376 | Regression_sin_phi: 0.0442 | Group Train Loss Unweighted (window mean): physical_pu: 3.6357 | physical_nopu: 4.4395 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5079 | Classification_binary: 3.3083 | Regression_cos_phi: 1.3180 | Regression_energy: 2.5583 | Regression_eta: 0.5953 | Regression_pt: 3.2423 | Regression_sin_phi: 1.6786 | LR: 1.43e-04 | DataLoad Time: 0.0027s | Model Forward Time: 6.4864s | Phase Times: max_pre_step_barrier: 0.1731 | max_batch_to_device: 0.0118 | max_forward: 0.1219 | max_loss: 0.0038 | max_backward: 0.1121 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 5.0441 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5600.0 | uss_MB: 4820.3 | child_rss_MB: 8744.3 | allocated_MB: 611.1 | reserved_MB: 29052.0 | inactive_split_MB: 372.9 +rank6 [2026-10-27 20:37:48,516] INFO : Step 65000/100000 rank6 | Train Loss: 16.3374 | Train Loss Unweighted: 4.8919 | Train Loss Components: Classification: 0.0142 | Classification_binary: 3.4871 | Regression_cos_phi: 0.0500 | Regression_energy: 0.6360 | Regression_eta: 0.0628 | Regression_pt: 0.5943 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4335 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4999 | Classification_binary: 3.5043 | Regression_cos_phi: 2.5189 | Regression_energy: 5.5449 | Regression_eta: 1.0882 | Regression_pt: 6.9437 | Regression_sin_phi: 2.6834 | LR: 1.38e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.4884s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.2167 | max_loss: 0.0058 | max_backward: 0.2407 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 1.9060 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5594.0 | uss_MB: 4820.8 | child_rss_MB: 8750.8 | allocated_MB: 609.7 | reserved_MB: 29052.0 | inactive_split_MB: 236.9 +rank6 [2026-10-27 20:41:37,390] INFO : Step 66000/100000 rank6 | Train Loss: 12.8848 | Train Loss Unweighted: 4.2935 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4715 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3451 | Regression_eta: 0.0693 | Regression_pt: 0.2907 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.6349 | physical_nopu: 4.4348 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4551 | Classification_binary: 3.5059 | Regression_cos_phi: 2.0218 | Regression_energy: 1.8982 | Regression_eta: 0.9023 | Regression_pt: 2.3177 | Regression_sin_phi: 1.6935 | LR: 1.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.1923s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0854 | max_loss: 0.1083 | max_backward: 0.2178 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.9218 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5582.1 | uss_MB: 4820.7 | child_rss_MB: 8750.8 | allocated_MB: 611.4 | reserved_MB: 29052.0 | inactive_split_MB: 254.2 +rank6 [2026-10-27 20:45:24,537] INFO : Step 67000/100000 rank6 | Train Loss: 14.2125 | Train Loss Unweighted: 4.5026 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.4493 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4566 | Regression_eta: 0.0671 | Regression_pt: 0.4150 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6352 | physical_nopu: 4.4350 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6958 | Classification_binary: 3.5348 | Regression_cos_phi: 2.4215 | Regression_energy: 1.9418 | Regression_eta: 1.1529 | Regression_pt: 2.4674 | Regression_sin_phi: 2.0222 | LR: 1.28e-04 | DataLoad Time: 0.0024s | Model Forward Time: 3.0120s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0901 | max_loss: 0.0487 | max_backward: 0.1449 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 1.8090 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5575.1 | uss_MB: 4820.8 | child_rss_MB: 8750.2 | allocated_MB: 607.1 | reserved_MB: 29260.0 | inactive_split_MB: 246.6 +rank6 [2026-10-27 20:49:05,126] INFO : Step 68000/100000 rank6 | Train Loss: 12.0487 | Train Loss Unweighted: 4.2633 | Train Loss Components: Classification: 0.0107 | Classification_binary: 3.5135 | Regression_cos_phi: 0.0466 | Regression_energy: 0.3165 | Regression_eta: 0.0645 | Regression_pt: 0.2647 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4387 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2838 | Classification_binary: 3.6078 | Regression_cos_phi: 1.9825 | Regression_energy: 1.3643 | Regression_eta: 1.2476 | Regression_pt: 1.3965 | Regression_sin_phi: 2.2198 | LR: 1.24e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.8889s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0010 | max_forward: 0.0879 | max_loss: 0.0120 | max_backward: 0.1069 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.7176 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5570.9 | uss_MB: 4820.5 | child_rss_MB: 8753.3 | allocated_MB: 609.5 | reserved_MB: 29260.0 | inactive_split_MB: 225.9 +rank6 [2026-10-27 20:52:47,291] INFO : Step 69000/100000 rank6 | Train Loss: 13.1377 | Train Loss Unweighted: 4.3133 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4496 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3632 | Regression_eta: 0.0778 | Regression_pt: 0.3079 | Regression_sin_phi: 0.0541 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4321 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5898 | Classification_binary: 3.4262 | Regression_cos_phi: 1.6850 | Regression_energy: 1.7210 | Regression_eta: 1.0091 | Regression_pt: 1.8148 | Regression_sin_phi: 2.0230 | LR: 1.19e-04 | DataLoad Time: 0.0015s | Model Forward Time: 4.9127s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.1207 | max_loss: 0.0509 | max_backward: 0.2012 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 3.5532 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 5571.9 | uss_MB: 4820.8 | child_rss_MB: 8755.8 | allocated_MB: 608.0 | reserved_MB: 29260.0 | inactive_split_MB: 256.4 +rank6 [2026-10-27 20:56:27,513] INFO : Step 70000/100000 rank6 | Train Loss: 15.8477 | Train Loss Unweighted: 4.7317 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3873 | Regression_cos_phi: 0.0498 | Regression_energy: 0.6015 | Regression_eta: 0.0629 | Regression_pt: 0.5683 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6315 | physical_nopu: 4.4280 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4393 | Classification_binary: 3.2639 | Regression_cos_phi: 1.8545 | Regression_energy: 2.6972 | Regression_eta: 0.7893 | Regression_pt: 3.4671 | Regression_sin_phi: 1.6863 | LR: 1.14e-04 | DataLoad Time: 0.0068s | Model Forward Time: 2.9278s | Phase Times: max_pre_step_barrier: 0.0198 | max_batch_to_device: 0.0007 | max_forward: 0.1178 | max_loss: 0.0036 | max_backward: 0.1081 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.7610 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5571.5 | uss_MB: 4819.7 | child_rss_MB: 8751.0 | allocated_MB: 610.3 | reserved_MB: 30150.0 | inactive_split_MB: 260.3 +rank6 [2026-10-27 20:56:28,457] INFO : Running validation on rank6 +rank6 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 20:58:25,954] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 20:58:26,359] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 20:58:52,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 20:58:52,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 20:58:53,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 20:58:53,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 20:58:54,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 20:58:54,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 20:58:55,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 20:58:55,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 20:59:05,703] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 20:59:05,946] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 20:59:06,380] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 20:59:21,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 20:59:21,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 20:59:22,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 20:59:23,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 20:59:23,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 20:59:24,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 20:59:24,579] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 20:59:25,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 20:59:35,089] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 20:59:35,862] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 20:59:36,279] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 20:59:51,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 20:59:51,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 20:59:52,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 20:59:52,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 20:59:53,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 20:59:53,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 20:59:54,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 20:59:54,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 21:00:04,809] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 21:04:25,115] INFO : Step 71000/100000 rank6 | Train Loss: 12.0110 | Train Loss Unweighted: 3.6539 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9046 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2881 | Regression_eta: 0.1063 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6306 | physical_nopu: 4.4256 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5462 | Classification_binary: 2.8129 | Regression_cos_phi: 1.7800 | Regression_energy: 1.6027 | Regression_eta: 1.7081 | Regression_pt: 1.8422 | Regression_sin_phi: 1.9873 | LR: 1.10e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.4585s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0035 | max_forward: 0.1952 | max_loss: 0.0038 | max_backward: 0.3195 | max_optimizer_step: 0.3092 | max_tensorboard_logging: 1.7411 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6798.7 | uss_MB: 6058.3 | child_rss_MB: 8722.6 | allocated_MB: 623.5 | reserved_MB: 27422.0 | inactive_split_MB: 864.2 +rank6 [2026-10-27 21:08:01,993] INFO : Step 72000/100000 rank6 | Train Loss: 15.2878 | Train Loss Unweighted: 4.6447 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.3643 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5735 | Regression_eta: 0.0647 | Regression_pt: 0.5353 | Regression_sin_phi: 0.0449 | Group Train Loss Unweighted (window mean): physical_pu: 3.6275 | physical_nopu: 4.4239 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4927 | Classification_binary: 3.4957 | Regression_cos_phi: 1.6909 | Regression_energy: 3.2919 | Regression_eta: 0.7210 | Regression_pt: 4.1133 | Regression_sin_phi: 1.5463 | LR: 1.05e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.5452s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0232 | max_forward: 0.1174 | max_loss: 0.0036 | max_backward: 0.0921 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 1.2606 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5641.1 | uss_MB: 4896.7 | child_rss_MB: 8682.2 | allocated_MB: 607.6 | reserved_MB: 28586.0 | inactive_split_MB: 303.9 +rank6 [2026-10-27 21:11:50,556] INFO : Step 73000/100000 rank6 | Train Loss: 12.7380 | Train Loss Unweighted: 4.2392 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.4842 | Regression_cos_phi: 0.0575 | Regression_energy: 0.3093 | Regression_eta: 0.0643 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.6309 | physical_nopu: 4.4270 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.1452 | Classification_binary: 3.2696 | Regression_cos_phi: 1.9114 | Regression_energy: 1.4329 | Regression_eta: 1.1387 | Regression_pt: 1.3386 | Regression_sin_phi: 1.8187 | LR: 1.01e-04 | DataLoad Time: 0.0007s | Model Forward Time: 2.4484s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0985 | max_loss: 0.0632 | max_backward: 0.1654 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2139 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5637.1 | uss_MB: 4897.0 | child_rss_MB: 8681.3 | allocated_MB: 607.0 | reserved_MB: 28586.0 | inactive_split_MB: 227.5 +rank6 [2026-10-27 21:15:34,267] INFO : Step 74000/100000 rank6 | Train Loss: 12.5735 | Train Loss Unweighted: 4.2478 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4076 | Regression_cos_phi: 0.0429 | Regression_energy: 0.3533 | Regression_eta: 0.0699 | Regression_pt: 0.3116 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6307 | physical_nopu: 4.4218 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5516 | Classification_binary: 3.4279 | Regression_cos_phi: 1.8563 | Regression_energy: 1.8657 | Regression_eta: 0.9496 | Regression_pt: 2.1634 | Regression_sin_phi: 1.7935 | LR: 9.68e-05 | DataLoad Time: 0.0026s | Model Forward Time: 2.5459s | Phase Times: max_pre_step_barrier: 0.1687 | max_batch_to_device: 0.0008 | max_forward: 0.0990 | max_loss: 0.0516 | max_backward: 0.1766 | max_optimizer_step: 0.1812 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5626.2 | uss_MB: 4897.3 | child_rss_MB: 8634.5 | allocated_MB: 608.6 | reserved_MB: 28586.0 | inactive_split_MB: 285.4 +rank6 [2026-10-27 21:19:21,651] INFO : Step 75000/100000 rank6 | Train Loss: 15.1687 | Train Loss Unweighted: 4.6098 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.3751 | Regression_cos_phi: 0.0492 | Regression_energy: 0.5530 | Regression_eta: 0.0638 | Regression_pt: 0.5060 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4206 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4508 | Classification_binary: 3.5133 | Regression_cos_phi: 2.1775 | Regression_energy: 2.3739 | Regression_eta: 1.0829 | Regression_pt: 2.7687 | Regression_sin_phi: 1.6705 | LR: 9.27e-05 | DataLoad Time: 0.0023s | Model Forward Time: 2.4832s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1026 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.2820 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5619.2 | uss_MB: 4896.7 | child_rss_MB: 8602.7 | allocated_MB: 610.1 | reserved_MB: 28586.0 | inactive_split_MB: 259.7 +rank6 [2026-10-27 21:23:05,981] INFO : Step 76000/100000 rank6 | Train Loss: 12.2943 | Train Loss Unweighted: 4.2773 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5085 | Regression_cos_phi: 0.0476 | Regression_energy: 0.3262 | Regression_eta: 0.0732 | Regression_pt: 0.2631 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6289 | physical_nopu: 4.4209 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4022 | Classification_binary: 3.4546 | Regression_cos_phi: 1.8220 | Regression_energy: 1.6177 | Regression_eta: 0.8786 | Regression_pt: 1.6885 | Regression_sin_phi: 2.0656 | LR: 8.88e-05 | DataLoad Time: 0.0020s | Model Forward Time: 2.6229s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.1085 | max_optimizer_step: 0.1831 | max_tensorboard_logging: 1.4262 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 5617.3 | uss_MB: 4895.6 | child_rss_MB: 8601.6 | allocated_MB: 608.9 | reserved_MB: 28942.0 | inactive_split_MB: 227.1 +rank6 [2026-10-27 21:27:01,963] INFO : Step 77000/100000 rank6 | Train Loss: 13.8477 | Train Loss Unweighted: 4.4496 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4225 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4510 | Regression_eta: 0.0623 | Regression_pt: 0.4032 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6274 | physical_nopu: 4.4166 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4995 | Classification_binary: 3.1201 | Regression_cos_phi: 1.3315 | Regression_energy: 2.4009 | Regression_eta: 0.4511 | Regression_pt: 2.9283 | Regression_sin_phi: 1.3326 | LR: 8.50e-05 | DataLoad Time: 0.0015s | Model Forward Time: 6.1308s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0839 | max_forward: 0.1754 | max_loss: 0.0036 | max_backward: 0.1103 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 4.6772 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1374 | Memory: rss_MB: 5607.3 | uss_MB: 4838.5 | child_rss_MB: 8502.5 | allocated_MB: 611.0 | reserved_MB: 28942.0 | inactive_split_MB: 369.0 +rank6 [2026-10-27 21:30:52,663] INFO : Step 78000/100000 rank6 | Train Loss: 14.1981 | Train Loss Unweighted: 4.6212 | Train Loss Components: Classification: 0.0133 | Classification_binary: 3.5747 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4573 | Regression_eta: 0.0669 | Regression_pt: 0.4114 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.6241 | physical_nopu: 4.4230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4497 | Classification_binary: 3.6843 | Regression_cos_phi: 2.3825 | Regression_energy: 1.9228 | Regression_eta: 1.1173 | Regression_pt: 2.0562 | Regression_sin_phi: 2.4587 | LR: 8.13e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.0985s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1149 | max_loss: 0.0636 | max_backward: 0.1914 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5607.3 | uss_MB: 4897.1 | child_rss_MB: 8495.1 | allocated_MB: 606.6 | reserved_MB: 28942.0 | inactive_split_MB: 309.2 +rank6 [2026-10-27 21:34:49,720] INFO : Step 79000/100000 rank6 | Train Loss: 12.3648 | Train Loss Unweighted: 4.2396 | Train Loss Components: Classification: 0.0145 | Classification_binary: 3.4636 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3226 | Regression_eta: 0.0628 | Regression_pt: 0.2792 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6254 | physical_nopu: 4.4183 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4050 | Classification_binary: 3.5045 | Regression_cos_phi: 1.8320 | Regression_energy: 1.6518 | Regression_eta: 1.1180 | Regression_pt: 1.8491 | Regression_sin_phi: 2.1005 | LR: 7.78e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.3388s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0936 | max_forward: 0.1936 | max_loss: 0.0037 | max_backward: 0.1134 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 2.0079 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 5602.4 | uss_MB: 4897.2 | child_rss_MB: 8483.1 | allocated_MB: 609.2 | reserved_MB: 29222.0 | inactive_split_MB: 324.6 +rank6 [2026-10-27 21:38:37,320] INFO : Step 80000/100000 rank6 | Train Loss: 16.1833 | Train Loss Unweighted: 4.8046 | Train Loss Components: Classification: 0.0155 | Classification_binary: 3.4149 | Regression_cos_phi: 0.0453 | Regression_energy: 0.6189 | Regression_eta: 0.0801 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6233 | physical_nopu: 4.4142 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4709 | Classification_binary: 3.2959 | Regression_cos_phi: 1.6542 | Regression_energy: 3.8147 | Regression_eta: 0.7632 | Regression_pt: 4.9113 | Regression_sin_phi: 1.9488 | LR: 7.44e-05 | DataLoad Time: 0.0092s | Model Forward Time: 3.9245s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0008 | max_forward: 0.1071 | max_loss: 0.0037 | max_backward: 0.1380 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 2.6784 | max_tensorboard_flush: 0.0026 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 5596.4 | uss_MB: 4871.2 | child_rss_MB: 8436.2 | allocated_MB: 606.3 | reserved_MB: 29222.0 | inactive_split_MB: 471.7 +rank6 [2026-10-27 21:38:38,322] INFO : Running validation on rank6 +rank6 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 21:40:39,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 21:40:39,981] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 21:41:05,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 21:41:06,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 21:41:06,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 21:41:07,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 21:41:07,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 21:41:08,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 21:41:08,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 21:41:09,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 21:41:19,239] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-27 21:41:19,914] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 21:41:20,297] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 21:41:35,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 21:41:36,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 21:41:36,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 21:41:37,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 21:41:37,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 21:41:38,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 21:41:38,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 21:41:39,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 21:41:49,360] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-27 21:41:49,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 21:41:50,005] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 21:42:05,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 21:42:05,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 21:42:06,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 21:42:06,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 21:42:07,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 21:42:07,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 21:42:08,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 21:42:08,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 21:42:18,839] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 21:46:50,145] INFO : Step 81000/100000 rank6 | Train Loss: 11.7500 | Train Loss Unweighted: 3.6065 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.8934 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2712 | Regression_eta: 0.1074 | Regression_pt: 0.2244 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4130 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5167 | Classification_binary: 2.9124 | Regression_cos_phi: 1.7196 | Regression_energy: 1.4339 | Regression_eta: 1.6054 | Regression_pt: 1.5659 | Regression_sin_phi: 1.8363 | LR: 7.11e-05 | DataLoad Time: 0.0020s | Model Forward Time: 8.6907s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0975 | max_forward: 0.3237 | max_loss: 0.0036 | max_backward: 0.5496 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 6.3197 | max_tensorboard_flush: 0.0011 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6963.5 | uss_MB: 4835.8 | child_rss_MB: 8415.8 | allocated_MB: 619.9 | reserved_MB: 28598.0 | inactive_split_MB: 1192.4 +rank6 [2026-10-27 21:50:33,876] INFO : Step 82000/100000 rank6 | Train Loss: 13.6704 | Train Loss Unweighted: 4.5604 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0419 | Regression_energy: 0.4618 | Regression_eta: 0.0599 | Regression_pt: 0.4291 | Regression_sin_phi: 0.0407 | Group Train Loss Unweighted (window mean): physical_pu: 3.6250 | physical_nopu: 4.4152 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5883 | Classification_binary: 3.4054 | Regression_cos_phi: 1.4406 | Regression_energy: 3.0215 | Regression_eta: 0.6530 | Regression_pt: 3.8551 | Regression_sin_phi: 1.2700 | LR: 6.80e-05 | DataLoad Time: 0.0014s | Model Forward Time: 3.1355s | Phase Times: max_pre_step_barrier: 0.1835 | max_batch_to_device: 0.0874 | max_forward: 0.1976 | max_loss: 0.0036 | max_backward: 0.1170 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 1.6260 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 5659.6 | uss_MB: 4976.9 | child_rss_MB: 8356.3 | allocated_MB: 607.5 | reserved_MB: 28598.0 | inactive_split_MB: 253.8 +rank6 [2026-10-27 21:54:19,227] INFO : Step 83000/100000 rank6 | Train Loss: 12.0270 | Train Loss Unweighted: 4.2414 | Train Loss Components: Classification: 0.0092 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0499 | Regression_energy: 0.3056 | Regression_eta: 0.0671 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4141 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2818 | Classification_binary: 3.6414 | Regression_cos_phi: 2.1291 | Regression_energy: 1.7871 | Regression_eta: 1.1412 | Regression_pt: 1.9913 | Regression_sin_phi: 2.1089 | LR: 6.51e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.7071s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1966 | max_loss: 0.0138 | max_backward: 0.1124 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 2.4026 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 5659.6 | uss_MB: 4977.1 | child_rss_MB: 8350.2 | allocated_MB: 609.6 | reserved_MB: 28598.0 | inactive_split_MB: 289.5 +rank6 [2026-10-27 21:58:08,515] INFO : Step 84000/100000 rank6 | Train Loss: 12.5172 | Train Loss Unweighted: 4.2412 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.4173 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3464 | Regression_eta: 0.0637 | Regression_pt: 0.3052 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4149 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6114 | Classification_binary: 3.3534 | Regression_cos_phi: 1.6379 | Regression_energy: 2.0060 | Regression_eta: 1.0514 | Regression_pt: 2.3654 | Regression_sin_phi: 1.8405 | LR: 6.23e-05 | DataLoad Time: 0.0014s | Model Forward Time: 3.2651s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1103 | max_loss: 0.0037 | max_backward: 0.1159 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 2.0319 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5655.6 | uss_MB: 4977.8 | child_rss_MB: 8312.0 | allocated_MB: 605.9 | reserved_MB: 28598.0 | inactive_split_MB: 208.8 +rank6 [2026-10-27 22:01:46,561] INFO : Step 85000/100000 rank6 | Train Loss: 15.2814 | Train Loss Unweighted: 4.6863 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.3811 | Regression_cos_phi: 0.0432 | Regression_energy: 0.5842 | Regression_eta: 0.0608 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6173 | physical_nopu: 4.4078 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5374 | Classification_binary: 3.4835 | Regression_cos_phi: 2.5876 | Regression_energy: 3.6531 | Regression_eta: 1.1246 | Regression_pt: 4.7548 | Regression_sin_phi: 1.8517 | LR: 5.96e-05 | DataLoad Time: 0.0010s | Model Forward Time: 3.0850s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.0993 | max_loss: 0.0558 | max_backward: 0.1762 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 1.8022 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 5645.3 | uss_MB: 4977.4 | child_rss_MB: 8242.1 | allocated_MB: 606.6 | reserved_MB: 28598.0 | inactive_split_MB: 257.4 +rank6 [2026-10-27 22:05:28,666] INFO : Step 86000/100000 rank6 | Train Loss: 12.8074 | Train Loss Unweighted: 4.3373 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.5785 | Regression_cos_phi: 0.0543 | Regression_energy: 0.3116 | Regression_eta: 0.0747 | Regression_pt: 0.2515 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.6255 | physical_nopu: 4.4086 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3172 | Classification_binary: 3.6213 | Regression_cos_phi: 1.8527 | Regression_energy: 1.7325 | Regression_eta: 1.7575 | Regression_pt: 1.7760 | Regression_sin_phi: 2.1845 | LR: 5.71e-05 | DataLoad Time: 0.0021s | Model Forward Time: 3.0437s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1215 | max_loss: 0.0037 | max_backward: 0.1229 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 1.6761 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5636.3 | uss_MB: 4977.9 | child_rss_MB: 8180.8 | allocated_MB: 608.3 | reserved_MB: 29688.0 | inactive_split_MB: 271.8 +rank6 [2026-10-27 22:09:22,619] INFO : Step 87000/100000 rank6 | Train Loss: 12.9093 | Train Loss Unweighted: 4.3003 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.3807 | Regression_cos_phi: 0.0447 | Regression_energy: 0.4014 | Regression_eta: 0.0615 | Regression_pt: 0.3523 | Regression_sin_phi: 0.0424 | Group Train Loss Unweighted (window mean): physical_pu: 3.6211 | physical_nopu: 4.4120 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5436 | Classification_binary: 3.4427 | Regression_cos_phi: 1.6350 | Regression_energy: 1.9770 | Regression_eta: 0.6766 | Regression_pt: 2.2726 | Regression_sin_phi: 1.7666 | LR: 5.48e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.0850s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0203 | max_forward: 0.1299 | max_loss: 0.0038 | max_backward: 0.1175 | max_optimizer_step: 0.2192 | max_tensorboard_logging: 1.8010 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5636.1 | uss_MB: 4575.9 | child_rss_MB: 8182.7 | allocated_MB: 607.0 | reserved_MB: 29688.0 | inactive_split_MB: 231.0 +rank6 [2026-10-27 22:13:02,799] INFO : Step 88000/100000 rank6 | Train Loss: 15.2369 | Train Loss Unweighted: 4.6456 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4525 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5240 | Regression_eta: 0.0665 | Regression_pt: 0.4840 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6192 | physical_nopu: 4.4119 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4067 | Classification_binary: 3.4255 | Regression_cos_phi: 1.9028 | Regression_energy: 3.9410 | Regression_eta: 1.0731 | Regression_pt: 5.0540 | Regression_sin_phi: 2.0036 | LR: 5.26e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.1173s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0007 | max_forward: 0.1127 | max_loss: 0.0236 | max_backward: 0.1542 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 1.7956 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5636.1 | uss_MB: 4977.3 | child_rss_MB: 8181.7 | allocated_MB: 609.0 | reserved_MB: 29688.0 | inactive_split_MB: 221.0 +rank6 [2026-10-27 22:16:52,919] INFO : Step 89000/100000 rank6 | Train Loss: 12.8609 | Train Loss Unweighted: 4.2765 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.4665 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3343 | Regression_eta: 0.0681 | Regression_pt: 0.2896 | Regression_sin_phi: 0.0555 | Group Train Loss Unweighted (window mean): physical_pu: 3.6177 | physical_nopu: 4.4110 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3791 | Classification_binary: 3.4242 | Regression_cos_phi: 1.6120 | Regression_energy: 1.4314 | Regression_eta: 0.7846 | Regression_pt: 1.7599 | Regression_sin_phi: 1.9376 | LR: 5.06e-05 | DataLoad Time: 0.0013s | Model Forward Time: 3.1965s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1194 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 1.9452 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 5632.5 | uss_MB: 4977.7 | child_rss_MB: 8137.7 | allocated_MB: 612.6 | reserved_MB: 30628.0 | inactive_split_MB: 309.7 +rank6 [2026-10-27 22:20:43,711] INFO : Step 90000/100000 rank6 | Train Loss: 11.6102 | Train Loss Unweighted: 3.6127 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9177 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2661 | Regression_eta: 0.1040 | Regression_pt: 0.2155 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4109 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4783 | Classification_binary: 2.8755 | Regression_cos_phi: 1.7603 | Regression_energy: 1.4579 | Regression_eta: 1.7299 | Regression_pt: 1.5478 | Regression_sin_phi: 1.8237 | LR: 4.88e-05 | DataLoad Time: 0.0025s | Model Forward Time: 3.5769s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0110 | max_forward: 0.1856 | max_loss: 0.0145 | max_backward: 0.4526 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 1.8620 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 5624.5 | uss_MB: 4985.7 | child_rss_MB: 8062.4 | allocated_MB: 617.2 | reserved_MB: 30628.0 | inactive_split_MB: 962.2 +rank6 [2026-10-27 22:20:44,690] INFO : Running validation on rank6 +rank6 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 22:22:45,204] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 22:22:45,606] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 22:23:10,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 22:23:11,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 22:23:11,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 22:23:12,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 22:23:12,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 22:23:13,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 22:23:13,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 22:23:14,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 22:23:24,060] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 22:23:24,365] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 22:23:24,734] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 22:23:39,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 22:23:40,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 22:23:40,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 22:23:41,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 22:23:41,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 22:23:42,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 22:23:42,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 22:23:43,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 22:23:53,211] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 22:23:53,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 22:23:54,273] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 22:24:09,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 22:24:10,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 22:24:10,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 22:24:11,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 22:24:11,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 22:24:12,397] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 22:24:12,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 22:24:13,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 22:24:23,387] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 22:28:41,211] INFO : Step 91000/100000 rank6 | Train Loss: 12.9063 | Train Loss Unweighted: 4.3371 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5236 | Regression_cos_phi: 0.0531 | Regression_energy: 0.3431 | Regression_eta: 0.0707 | Regression_pt: 0.2829 | Regression_sin_phi: 0.0512 | Group Train Loss Unweighted (window mean): physical_pu: 3.6180 | physical_nopu: 4.4087 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3528 | Classification_binary: 3.2912 | Regression_cos_phi: 1.3817 | Regression_energy: 1.7981 | Regression_eta: 0.5710 | Regression_pt: 2.2439 | Regression_sin_phi: 1.5901 | LR: 4.71e-05 | DataLoad Time: 0.0020s | Model Forward Time: 4.8999s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0055 | max_forward: 0.1521 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 3.4850 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7066.6 | uss_MB: 6432.9 | child_rss_MB: 8020.4 | allocated_MB: 607.8 | reserved_MB: 28796.0 | inactive_split_MB: 320.2 +rank6 [2026-10-27 22:32:23,431] INFO : Step 92000/100000 rank6 | Train Loss: 15.4341 | Train Loss Unweighted: 4.7247 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4285 | Regression_cos_phi: 0.0457 | Regression_energy: 0.5811 | Regression_eta: 0.0735 | Regression_pt: 0.5375 | Regression_sin_phi: 0.0425 | Group Train Loss Unweighted (window mean): physical_pu: 3.6150 | physical_nopu: 4.4032 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3994 | Classification_binary: 3.4086 | Regression_cos_phi: 1.5369 | Regression_energy: 3.4994 | Regression_eta: 1.0754 | Regression_pt: 4.2682 | Regression_sin_phi: 1.3763 | LR: 4.57e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.3948s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.2696 | max_loss: 0.0037 | max_backward: 0.1715 | max_optimizer_step: 0.3237 | max_tensorboard_logging: 1.7768 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7051.6 | uss_MB: 6432.9 | child_rss_MB: 7931.4 | allocated_MB: 609.3 | reserved_MB: 28796.0 | inactive_split_MB: 222.8 +rank6 [2026-10-27 22:36:08,796] INFO : Step 93000/100000 rank6 | Train Loss: 13.9744 | Train Loss Unweighted: 4.4854 | Train Loss Components: Classification: 0.0118 | Classification_binary: 3.5124 | Regression_cos_phi: 0.0578 | Regression_energy: 0.4245 | Regression_eta: 0.0652 | Regression_pt: 0.3624 | Regression_sin_phi: 0.0513 | Group Train Loss Unweighted (window mean): physical_pu: 3.6205 | physical_nopu: 4.4075 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3894 | Classification_binary: 3.7994 | Regression_cos_phi: 2.0778 | Regression_energy: 2.1914 | Regression_eta: 1.1465 | Regression_pt: 2.3748 | Regression_sin_phi: 2.1007 | LR: 4.43e-05 | DataLoad Time: 0.0022s | Model Forward Time: 2.9849s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1098 | max_loss: 0.0036 | max_backward: 0.1173 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 1.7709 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5706.9 | uss_MB: 5092.9 | child_rss_MB: 7894.0 | allocated_MB: 606.4 | reserved_MB: 30394.0 | inactive_split_MB: 244.2 +rank6 [2026-10-27 22:39:53,036] INFO : Step 94000/100000 rank6 | Train Loss: 12.4397 | Train Loss Unweighted: 4.2538 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4230 | Regression_cos_phi: 0.0415 | Regression_energy: 0.3513 | Regression_eta: 0.0702 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6164 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5660 | Classification_binary: 3.4809 | Regression_cos_phi: 1.6885 | Regression_energy: 2.0523 | Regression_eta: 1.3273 | Regression_pt: 2.0893 | Regression_sin_phi: 1.9669 | LR: 4.32e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.1516s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0009 | max_forward: 0.1540 | max_loss: 0.0036 | max_backward: 0.2309 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 1.7723 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 5695.9 | uss_MB: 4753.0 | child_rss_MB: 7806.9 | allocated_MB: 607.1 | reserved_MB: 30394.0 | inactive_split_MB: 232.2 +rank6 [2026-10-27 22:43:36,954] INFO : Step 95000/100000 rank6 | Train Loss: 14.9177 | Train Loss Unweighted: 4.6108 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3911 | Regression_cos_phi: 0.0436 | Regression_energy: 0.5612 | Regression_eta: 0.0598 | Regression_pt: 0.4905 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6121 | physical_nopu: 4.4076 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4645 | Classification_binary: 3.3864 | Regression_cos_phi: 1.2718 | Regression_energy: 2.3428 | Regression_eta: 0.7330 | Regression_pt: 2.9045 | Regression_sin_phi: 1.3717 | LR: 4.22e-05 | DataLoad Time: 0.0015s | Model Forward Time: 3.0357s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1111 | max_loss: 0.0199 | max_backward: 0.1395 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.7978 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5695.9 | uss_MB: 5093.1 | child_rss_MB: 7811.5 | allocated_MB: 609.0 | reserved_MB: 30394.0 | inactive_split_MB: 285.3 +rank6 [2026-10-27 22:47:14,355] INFO : Step 96000/100000 rank6 | Train Loss: 12.4738 | Train Loss Unweighted: 4.2244 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.4884 | Regression_cos_phi: 0.0540 | Regression_energy: 0.2967 | Regression_eta: 0.0804 | Regression_pt: 0.2429 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6137 | physical_nopu: 4.4081 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2866 | Classification_binary: 3.6680 | Regression_cos_phi: 2.4882 | Regression_energy: 2.3239 | Regression_eta: 1.5283 | Regression_pt: 2.5742 | Regression_sin_phi: 2.2322 | LR: 4.14e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.1608s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0007 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.2003 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 1.8675 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5695.9 | uss_MB: 5093.2 | child_rss_MB: 7804.9 | allocated_MB: 608.1 | reserved_MB: 30394.0 | inactive_split_MB: 259.4 +rank6 [2026-10-27 22:50:52,983] INFO : Step 97000/100000 rank6 | Train Loss: 12.7922 | Train Loss Unweighted: 4.2852 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.4355 | Regression_cos_phi: 0.0455 | Regression_energy: 0.3604 | Regression_eta: 0.0703 | Regression_pt: 0.3091 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6136 | physical_nopu: 4.4015 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5215 | Classification_binary: 3.4873 | Regression_cos_phi: 1.7399 | Regression_energy: 2.1313 | Regression_eta: 1.2431 | Regression_pt: 2.1874 | Regression_sin_phi: 1.9143 | LR: 4.08e-05 | DataLoad Time: 0.0026s | Model Forward Time: 3.0102s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1110 | max_loss: 0.0174 | max_backward: 0.1239 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 1.7282 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0908 | Memory: rss_MB: 5687.4 | uss_MB: 5092.4 | child_rss_MB: 7746.9 | allocated_MB: 607.2 | reserved_MB: 30394.0 | inactive_split_MB: 265.3 +rank6 [2026-10-27 22:54:49,599] INFO : Step 98000/100000 rank6 | Train Loss: 13.8508 | Train Loss Unweighted: 4.4512 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3736 | Regression_cos_phi: 0.0443 | Regression_energy: 0.4792 | Regression_eta: 0.0606 | Regression_pt: 0.4354 | Regression_sin_phi: 0.0433 | Group Train Loss Unweighted (window mean): physical_pu: 3.6169 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4888 | Classification_binary: 3.3015 | Regression_cos_phi: 1.6887 | Regression_energy: 1.9987 | Regression_eta: 0.8564 | Regression_pt: 2.2926 | Regression_sin_phi: 1.7538 | LR: 4.04e-05 | DataLoad Time: 0.0025s | Model Forward Time: 3.0652s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0119 | max_forward: 0.1317 | max_loss: 0.0036 | max_backward: 0.1246 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.7312 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0878 | Memory: rss_MB: 5687.4 | uss_MB: 5093.6 | child_rss_MB: 7731.7 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 292.7 +rank6 [2026-10-27 22:58:30,099] INFO : Step 99000/100000 rank6 | Train Loss: 12.5729 | Train Loss Unweighted: 4.3128 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5536 | Regression_cos_phi: 0.0519 | Regression_energy: 0.3097 | Regression_eta: 0.0644 | Regression_pt: 0.2679 | Regression_sin_phi: 0.0529 | Group Train Loss Unweighted (window mean): physical_pu: 3.6154 | physical_nopu: 4.4035 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4420 | Classification_binary: 3.5954 | Regression_cos_phi: 1.7820 | Regression_energy: 2.0731 | Regression_eta: 1.0816 | Regression_pt: 2.3972 | Regression_sin_phi: 2.8717 | LR: 4.01e-05 | DataLoad Time: 0.0113s | Model Forward Time: 3.0444s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0008 | max_forward: 0.1158 | max_loss: 0.0113 | max_backward: 0.1168 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 1.7726 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 5687.4 | uss_MB: 5093.5 | child_rss_MB: 7727.1 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 283.7 +rank6 [2026-10-27 23:02:17,099] INFO : Step 100000/100000 rank6 | Train Loss: 11.9488 | Train Loss Unweighted: 3.6450 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9057 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2836 | Regression_eta: 0.1038 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6201 | physical_nopu: 4.4070 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4863 | Classification_binary: 2.8476 | Regression_cos_phi: 1.8127 | Regression_energy: 1.5278 | Regression_eta: 1.5539 | Regression_pt: 1.7290 | Regression_sin_phi: 1.8901 | LR: 4.00e-05 | DataLoad Time: 0.0034s | Model Forward Time: 6.3801s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0031 | max_forward: 0.2860 | max_loss: 0.0036 | max_backward: 0.4897 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 4.4601 | max_tensorboard_flush: 0.0009 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5687.2 | uss_MB: 4556.1 | child_rss_MB: 7711.3 | allocated_MB: 618.6 | reserved_MB: 30394.0 | inactive_split_MB: 1112.1 +rank6 [2026-10-27 23:02:18,080] INFO : Running validation on rank6 +rank6 [2026-10-27 23:02:18,081] INFO : ResumableSampler physical_pu:valid resetting +rank6 [2026-10-27 23:02:18,081] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank6 [2026-10-27 23:02:18,081] INFO : ResumableSampler physical_nopu:valid resetting +rank6 [2026-10-27 23:02:18,081] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank6 [2026-10-27 23:04:23,625] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 23:04:24,087] INFO : Running predictions on cms_pf_ttbar +rank6 [2026-10-27 23:04:49,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_6_0.parquet +rank6 [2026-10-27 23:04:50,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_6_1.parquet +rank6 [2026-10-27 23:04:50,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_6_2.parquet +rank6 [2026-10-27 23:04:51,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_6_3.parquet +rank6 [2026-10-27 23:04:51,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_6_4.parquet +rank6 [2026-10-27 23:04:52,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_6_5.parquet +rank6 [2026-10-27 23:04:52,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_6_6.parquet +rank6 [2026-10-27 23:04:53,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_6_7.parquet +rank6 [2026-10-27 23:05:03,299] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 23:05:03,812] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 23:05:04,178] INFO : Running predictions on cms_pf_qcd +rank6 [2026-10-27 23:05:19,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_6_0.parquet +rank6 [2026-10-27 23:05:20,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_6_1.parquet +rank6 [2026-10-27 23:05:20,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_6_2.parquet +rank6 [2026-10-27 23:05:21,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_6_3.parquet +rank6 [2026-10-27 23:05:21,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_6_4.parquet +rank6 [2026-10-27 23:05:22,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_6_5.parquet +rank6 [2026-10-27 23:05:22,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_6_6.parquet +rank6 [2026-10-27 23:05:23,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_6_7.parquet +rank6 [2026-10-27 23:05:33,107] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-27 23:05:33,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-27 23:05:34,200] INFO : Running predictions on cms_pf_ztt +rank6 [2026-10-27 23:05:49,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_6_0.parquet +rank6 [2026-10-27 23:05:50,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_6_1.parquet +rank6 [2026-10-27 23:05:50,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_6_2.parquet +rank6 [2026-10-27 23:05:51,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_6_3.parquet +rank6 [2026-10-27 23:05:51,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_6_4.parquet +rank6 [2026-10-27 23:05:52,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_6_5.parquet +rank6 [2026-10-27 23:05:52,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_6_6.parquet +rank6 [2026-10-27 23:05:53,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_6_7.parquet +rank6 [2026-10-27 23:06:03,391] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-27 23:06:03,966] INFO : Training completed. Total time on device 6: 414.788min diff --git a/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.7 b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.7 new file mode 100644 index 0000000000000000000000000000000000000000..388fda04e95490a769cbd0fececdbfb06b2429d0 --- /dev/null +++ b/cms/v3.2.0/heptv2_seed12345_20261027_160940_124470/train.log.7 @@ -0,0 +1,897 @@ +rank7 [2026-10-27 16:10:10,903] INFO : Initializing model with seed=12345; process seed=12352 +rank7 [2026-10-27 16:10:10,904] INFO : configured dtype=torch.bfloat16 for autocast +rank7 [2026-10-27 16:10:35,267] INFO : Instantiating model +rank7 [2026-10-27 16:10:35,267] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=3 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=55 engineered_input_dim=55 +rank7 [2026-10-27 16:10:35,267] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 4, 5, 6, 8, 9, 10, 11] +rank7 [2026-10-27 16:10:35,296] INFO : Input encoding initialization took 0.03s +rank7 [2026-10-27 16:10:35,296] INFO : nn0 parameters: 2130944 +rank7 [2026-10-27 16:10:35,296] INFO : Initializing shared backbone layers, num_convs=3 +rank7 [2026-10-27 16:10:35,296] INFO : Initializing backbone layer 0 +rank7 [2026-10-27 16:10:35,350] INFO : Initializing backbone layer 1 +rank7 [2026-10-27 16:10:35,380] INFO : Initializing backbone layer 2 +rank7 [2026-10-27 16:10:35,410] INFO : Backbone initialization took 0.11s +rank7 [2026-10-27 16:10:35,410] INFO : backbone parameters: 15765696 +rank7 [2026-10-27 16:10:35,410] INFO : Initializing output DNNs +rank7 [2026-10-27 16:10:35,444] INFO : Output DNNs initialization took 0.03s +rank7 [2026-10-27 16:10:35,444] INFO : backbone_mode=BackboneMode.SHARED +rank7 [2026-10-27 16:10:35,444] INFO : jagged_attention enabled=False +rank7 [2026-10-27 16:10:35,444] INFO : flash_attn_varlen enabled=False +rank7 [2026-10-27 16:10:35,445] INFO : task_queries enabled=False +rank7 [2026-10-27 16:10:35,445] INFO : classification_norm parameters: 0 +rank7 [2026-10-27 16:10:35,445] INFO : regression_norm parameters: 0 +rank7 [2026-10-27 16:10:35,445] INFO : classification_query parameters: 0 +rank7 [2026-10-27 16:10:35,445] INFO : regression_query parameters: 0 +rank7 [2026-10-27 16:10:35,445] INFO : classification_readout parameters: 0 +rank7 [2026-10-27 16:10:35,445] INFO : regression_readout parameters: 0 +rank7 [2026-10-27 16:10:35,445] INFO : output_mode=elementwise +rank7 [2026-10-27 16:10:35,445] INFO : set_decoder parameters: 0 +rank7 [2026-10-27 16:10:35,445] INFO : nn_binary_particle parameters: 264706 +rank7 [2026-10-27 16:10:35,445] INFO : nn_pid parameters: 268297 +rank7 [2026-10-27 16:10:35,445] INFO : nn_pt parameters: 2113544 +rank7 [2026-10-27 16:10:35,445] INFO : nn_eta parameters: 264706 +rank7 [2026-10-27 16:10:35,445] INFO : nn_sin_phi parameters: 264706 +rank7 [2026-10-27 16:10:35,445] INFO : nn_cos_phi parameters: 264706 +rank7 [2026-10-27 16:10:35,446] INFO : nn_energy parameters: 2113544 +rank7 [2026-10-27 16:10:35,446] INFO : Total MLPF parameters: 23450849 +rank7 [2026-10-27 16:10:35,446] INFO : MLPF __init__ done +rank7 [2026-10-27 16:10:35,446] INFO : Instantiated model +rank7 [2026-10-27 16:10:35,446] INFO : Moving model to device rank=7 +rank7 [2026-10-27 16:10:35,478] INFO : Moved model to device rank=7 +rank7 [2026-10-27 16:10:35,480] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0004 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank7 [2026-10-27 16:10:35,480] INFO : Configured model for SyncBatchNorm rank=7 +rank7 [2026-10-27 16:10:36,005] INFO : Configured model for DistributedDataParallel rank=7 +rank7 [2026-10-27 16:10:36,123] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +202 module.nn_energy.nn.7.0.bias ... 0 +203 module.nn_energy.nn.7.2.weight ... 0 +204 module.nn_energy.nn.7.2.bias ... 0 +205 module.nn_energy.nn.7.4.weight ... 0 +206 module.nn_energy.nn.7.4.bias ... 0 + +[207 rows x 3 columns] +rank7 [2026-10-27 16:10:36,123] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=55, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=4096, bias=True) + ) + (backbone): ModuleList( + (0-2): 3 x HEPTv2Layer( + embedding_dim=512, num_heads=16, width=512, dropout=0.0, block_size=100, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=512, out_features=512, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=512, out_features=512, bias=False) + (w_k): Linear(in_features=512, out_features=512, bias=False) + (w_v): Linear(in_features=512, out_features=512, bias=False) + (w_rpe): Linear(in_features=10, out_features=512, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=2048, bias=True) + (1): LayerNorm((2048,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=2048, out_features=512, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=512, out_features=2048, bias=False) + (up_proj): Linear(in_features=512, out_features=2048, bias=False) + (down_proj): Linear(in_features=2048, out_features=512, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (nn_binary_particle): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=9, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-7): 8 x Sequential( + (0): Linear(in_features=512, out_features=512, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((512,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=512, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank7 [2026-10-27 16:10:36,124] INFO : Trainable parameters: 23450849 +rank7 [2026-10-27 16:10:36,124] INFO : Non-trainable parameters: 22386 +rank7 [2026-10-27 16:10:36,124] INFO : Total parameters: 23473235 +rank7 [2026-10-27 16:10:36,126] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 28160 0 + module.nn0.0.bias 512 0 + module.nn0.2.weight 512 0 + module.nn0.2.bias 512 0 + module.nn0.4.weight 2097152 0 + module.nn0.4.bias 4096 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 262144 0 + module.backbone.0.attn.out_linear.bias 512 0 + module.backbone.0.attn.e2lsh.alpha 0 1632 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 262144 0 + module.backbone.0.w_k.weight 262144 0 + module.backbone.0.w_v.weight 262144 0 + module.backbone.0.w_rpe.weight 0 5120 + module.backbone.0.w_rpe.bias 0 512 +module.backbone.0.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.0.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.0.norm1.weight 512 0 + module.backbone.0.norm2.weight 512 0 + module.backbone.0.q_norm.weight 32 0 + module.backbone.0.k_norm.weight 32 0 + module.backbone.0.ff.gate_proj.weight 1048576 0 + module.backbone.0.ff.up_proj.weight 1048576 0 + module.backbone.0.ff.down_proj.weight 1048576 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 262144 0 + module.backbone.1.attn.out_linear.bias 512 0 + module.backbone.1.attn.e2lsh.alpha 0 1632 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 262144 0 + module.backbone.1.w_k.weight 262144 0 + module.backbone.1.w_v.weight 262144 0 + module.backbone.1.w_rpe.weight 0 5120 + module.backbone.1.w_rpe.bias 0 512 +module.backbone.1.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.1.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.1.norm1.weight 512 0 + module.backbone.1.norm2.weight 512 0 + module.backbone.1.q_norm.weight 32 0 + module.backbone.1.k_norm.weight 32 0 + module.backbone.1.ff.gate_proj.weight 1048576 0 + module.backbone.1.ff.up_proj.weight 1048576 0 + module.backbone.1.ff.down_proj.weight 1048576 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 262144 0 + module.backbone.2.attn.out_linear.bias 512 0 + module.backbone.2.attn.e2lsh.alpha 0 1632 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 262144 0 + module.backbone.2.w_k.weight 262144 0 + module.backbone.2.w_v.weight 262144 0 + module.backbone.2.w_rpe.weight 0 5120 + module.backbone.2.w_rpe.bias 0 512 +module.backbone.2.pe_func.position_embedding_head.0.weight 4096 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.1.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 2048 0 +module.backbone.2.pe_func.position_embedding_head.3.weight 1048576 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 512 0 + module.backbone.2.norm1.weight 512 0 + module.backbone.2.norm2.weight 512 0 + module.backbone.2.q_norm.weight 32 0 + module.backbone.2.k_norm.weight 32 0 + module.backbone.2.ff.gate_proj.weight 1048576 0 + module.backbone.2.ff.up_proj.weight 1048576 0 + module.backbone.2.ff.down_proj.weight 1048576 0 + module.nn_binary_particle.0.weight 262144 0 + module.nn_binary_particle.0.bias 512 0 + module.nn_binary_particle.2.weight 512 0 + module.nn_binary_particle.2.bias 512 0 + module.nn_binary_particle.4.weight 1024 0 + module.nn_binary_particle.4.bias 2 0 + module.nn_pid.0.weight 262144 0 + module.nn_pid.0.bias 512 0 + module.nn_pid.2.weight 512 0 + module.nn_pid.2.bias 512 0 + module.nn_pid.4.weight 4608 0 + module.nn_pid.4.bias 9 0 + module.nn_pt.nn.0.0.weight 262144 0 + module.nn_pt.nn.0.0.bias 512 0 + module.nn_pt.nn.0.2.weight 512 0 + module.nn_pt.nn.0.2.bias 512 0 + module.nn_pt.nn.0.4.weight 512 0 + module.nn_pt.nn.0.4.bias 1 0 + module.nn_pt.nn.1.0.weight 262144 0 + module.nn_pt.nn.1.0.bias 512 0 + module.nn_pt.nn.1.2.weight 512 0 + module.nn_pt.nn.1.2.bias 512 0 + module.nn_pt.nn.1.4.weight 512 0 + module.nn_pt.nn.1.4.bias 1 0 + module.nn_pt.nn.2.0.weight 262144 0 + module.nn_pt.nn.2.0.bias 512 0 + module.nn_pt.nn.2.2.weight 512 0 + module.nn_pt.nn.2.2.bias 512 0 + module.nn_pt.nn.2.4.weight 512 0 + module.nn_pt.nn.2.4.bias 1 0 + module.nn_pt.nn.3.0.weight 262144 0 + module.nn_pt.nn.3.0.bias 512 0 + module.nn_pt.nn.3.2.weight 512 0 + module.nn_pt.nn.3.2.bias 512 0 + module.nn_pt.nn.3.4.weight 512 0 + module.nn_pt.nn.3.4.bias 1 0 + module.nn_pt.nn.4.0.weight 262144 0 + module.nn_pt.nn.4.0.bias 512 0 + module.nn_pt.nn.4.2.weight 512 0 + module.nn_pt.nn.4.2.bias 512 0 + module.nn_pt.nn.4.4.weight 512 0 + module.nn_pt.nn.4.4.bias 1 0 + module.nn_pt.nn.5.0.weight 262144 0 + module.nn_pt.nn.5.0.bias 512 0 + module.nn_pt.nn.5.2.weight 512 0 + module.nn_pt.nn.5.2.bias 512 0 + module.nn_pt.nn.5.4.weight 512 0 + module.nn_pt.nn.5.4.bias 1 0 + module.nn_pt.nn.6.0.weight 262144 0 + module.nn_pt.nn.6.0.bias 512 0 + module.nn_pt.nn.6.2.weight 512 0 + module.nn_pt.nn.6.2.bias 512 0 + module.nn_pt.nn.6.4.weight 512 0 + module.nn_pt.nn.6.4.bias 1 0 + module.nn_pt.nn.7.0.weight 262144 0 + module.nn_pt.nn.7.0.bias 512 0 + module.nn_pt.nn.7.2.weight 512 0 + module.nn_pt.nn.7.2.bias 512 0 + module.nn_pt.nn.7.4.weight 512 0 + module.nn_pt.nn.7.4.bias 1 0 + module.nn_eta.nn.0.weight 262144 0 + module.nn_eta.nn.0.bias 512 0 + module.nn_eta.nn.2.weight 512 0 + module.nn_eta.nn.2.bias 512 0 + module.nn_eta.nn.4.weight 1024 0 + module.nn_eta.nn.4.bias 2 0 + module.nn_sin_phi.nn.0.weight 262144 0 + module.nn_sin_phi.nn.0.bias 512 0 + module.nn_sin_phi.nn.2.weight 512 0 + module.nn_sin_phi.nn.2.bias 512 0 + module.nn_sin_phi.nn.4.weight 1024 0 + module.nn_sin_phi.nn.4.bias 2 0 + module.nn_cos_phi.nn.0.weight 262144 0 + module.nn_cos_phi.nn.0.bias 512 0 + module.nn_cos_phi.nn.2.weight 512 0 + module.nn_cos_phi.nn.2.bias 512 0 + module.nn_cos_phi.nn.4.weight 1024 0 + module.nn_cos_phi.nn.4.bias 2 0 + module.nn_energy.nn.0.0.weight 262144 0 + module.nn_energy.nn.0.0.bias 512 0 + module.nn_energy.nn.0.2.weight 512 0 + module.nn_energy.nn.0.2.bias 512 0 + module.nn_energy.nn.0.4.weight 512 0 + module.nn_energy.nn.0.4.bias 1 0 + module.nn_energy.nn.1.0.weight 262144 0 + module.nn_energy.nn.1.0.bias 512 0 + module.nn_energy.nn.1.2.weight 512 0 + module.nn_energy.nn.1.2.bias 512 0 + module.nn_energy.nn.1.4.weight 512 0 + module.nn_energy.nn.1.4.bias 1 0 + module.nn_energy.nn.2.0.weight 262144 0 + module.nn_energy.nn.2.0.bias 512 0 + module.nn_energy.nn.2.2.weight 512 0 + module.nn_energy.nn.2.2.bias 512 0 + module.nn_energy.nn.2.4.weight 512 0 + module.nn_energy.nn.2.4.bias 1 0 + module.nn_energy.nn.3.0.weight 262144 0 + module.nn_energy.nn.3.0.bias 512 0 + module.nn_energy.nn.3.2.weight 512 0 + module.nn_energy.nn.3.2.bias 512 0 + module.nn_energy.nn.3.4.weight 512 0 + module.nn_energy.nn.3.4.bias 1 0 + module.nn_energy.nn.4.0.weight 262144 0 + module.nn_energy.nn.4.0.bias 512 0 + module.nn_energy.nn.4.2.weight 512 0 + module.nn_energy.nn.4.2.bias 512 0 + module.nn_energy.nn.4.4.weight 512 0 + module.nn_energy.nn.4.4.bias 1 0 + module.nn_energy.nn.5.0.weight 262144 0 + module.nn_energy.nn.5.0.bias 512 0 + module.nn_energy.nn.5.2.weight 512 0 + module.nn_energy.nn.5.2.bias 512 0 + module.nn_energy.nn.5.4.weight 512 0 + module.nn_energy.nn.5.4.bias 1 0 + module.nn_energy.nn.6.0.weight 262144 0 + module.nn_energy.nn.6.0.bias 512 0 + module.nn_energy.nn.6.2.weight 512 0 + module.nn_energy.nn.6.2.bias 512 0 + module.nn_energy.nn.6.4.weight 512 0 + module.nn_energy.nn.6.4.bias 1 0 + module.nn_energy.nn.7.0.weight 262144 0 + module.nn_energy.nn.7.0.bias 512 0 + module.nn_energy.nn.7.2.weight 512 0 + module.nn_energy.nn.7.2.bias 512 0 + module.nn_energy.nn.7.4.weight 512 0 + module.nn_energy.nn.7.4.bias 1 0 +rank7 [2026-10-27 16:10:36,126] INFO : Getting dataloaders +rank7 [2026-10-27 16:10:36,126] INFO : sample=cms_pf_ttbar split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:43,313] INFO : sample=cms_pf_qcd split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:44,043] INFO : sample=cms_pf_ztt split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:44,873] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank7 [2026-10-27 16:10:44,874] INFO : sample=cms_pf_ttbar_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:45,827] INFO : sample=cms_pf_qcd_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:46,992] INFO : sample=cms_pf_ztt_nopu split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:47,782] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank7 [2026-10-27 16:10:47,782] INFO : Creating WeightedMixIterator with 2 data loaders. +rank7 [2026-10-27 16:10:47,783] INFO : sample=cms_pf_ttbar split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:47,942] INFO : sample=cms_pf_qcd split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:48,367] INFO : sample=cms_pf_ztt split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:48,513] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank7 [2026-10-27 16:10:48,513] INFO : sample=cms_pf_ttbar_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:48,687] INFO : sample=cms_pf_qcd_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:48,845] INFO : sample=cms_pf_ztt_nopu split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:10:48,976] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank7 [2026-10-27 16:10:48,977] INFO : Creating InterleavedIterator with 2 data loaders. +rank7 [2026-10-27 16:11:16,659] INFO : Got dataloaders +rank7 [2026-10-27 16:11:16,659] INFO : loader split=train rank=7 len=232058 +rank7 [2026-10-27 16:11:16,659] INFO : loader split=valid rank=7 len=48 +rank7 [2026-10-27 16:11:16,660] INFO : Starting training from step 1 to 100000 on rank 7 of 8 +rank7 [2026-10-27 16:11:16,660] INFO : Creating train_iterator +rank7 [2026-10-27 16:11:16,660] INFO : ResumableSampler physical_pu:train loading state: start_index=0 +rank7 [2026-10-27 16:11:30,683] INFO : ResumableSampler physical_nopu:train loading state: start_index=0 +rank7 [2026-10-27 16:11:45,959] INFO : Created train_iterator +rank7 [2026-10-27 16:12:32,257] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.129867712855339, 'Classification': 0.16580097352620216, 'Regression_pt': 0.7168090478330851, 'Regression_eta': 0.335200086296536, 'Regression_sin_phi': 0.1274821354681626, 'Regression_cos_phi': 0.1371471309568733, 'Regression_energy': 0.9554258581250906}, weights={'Classification_binary': 1.0, 'Classification': 30.939912796020508, 'Regression_pt': 7.156533241271973, 'Regression_eta': 15.3038969039917, 'Regression_sin_phi': 40.23989486694336, 'Regression_cos_phi': 37.40412139892578, 'Regression_energy': 5.369194984436035} +rank7 [2026-10-27 16:15:18,322] INFO : Step 1000/100000 rank7 | Train Loss: 16.8632 | Train Loss Unweighted: 5.4934 | Train Loss Components: Classification: 0.0189 | Classification_binary: 4.1833 | Regression_cos_phi: 0.0509 | Regression_energy: 0.5806 | Regression_eta: 0.0776 | Regression_pt: 0.5302 | Regression_sin_phi: 0.0519 | Group Train Loss Unweighted (window mean): physical_pu: 5.1301 | physical_nopu: 6.0772 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5713 | Classification_binary: 4.1148 | Regression_cos_phi: 2.8186 | Regression_energy: 2.3211 | Regression_eta: 1.0398 | Regression_pt: 2.7107 | Regression_sin_phi: 1.8799 | LR: 4.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.3139s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0014 | max_forward: 0.0387 | max_loss: 0.0036 | max_backward: 0.0664 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 1.2817 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 4374.6 | uss_MB: 3102.3 | child_rss_MB: 11636.1 | allocated_MB: 609.1 | reserved_MB: 27788.0 | inactive_split_MB: 244.1 +rank7 [2026-10-27 16:18:42,006] INFO : Step 2000/100000 rank7 | Train Loss: 17.7204 | Train Loss Unweighted: 5.4242 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.9240 | Regression_cos_phi: 0.0505 | Regression_energy: 0.6758 | Regression_eta: 0.0665 | Regression_pt: 0.6366 | Regression_sin_phi: 0.0551 | Group Train Loss Unweighted (window mean): physical_pu: 4.2802 | physical_nopu: 5.3470 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4881 | Classification_binary: 3.8359 | Regression_cos_phi: 1.8241 | Regression_energy: 5.1538 | Regression_eta: 1.0161 | Regression_pt: 6.6037 | Regression_sin_phi: 1.5887 | LR: 4.00e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.8060s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1422 | max_loss: 0.0036 | max_backward: 0.1717 | max_optimizer_step: 0.1501 | max_tensorboard_logging: 0.7287 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 4367.5 | uss_MB: 3102.8 | child_rss_MB: 11637.6 | allocated_MB: 607.0 | reserved_MB: 27788.0 | inactive_split_MB: 326.9 +rank7 [2026-10-27 16:22:10,507] INFO : Step 3000/100000 rank7 | Train Loss: 14.3102 | Train Loss Unweighted: 4.8875 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.8970 | Regression_cos_phi: 0.0491 | Regression_energy: 0.4277 | Regression_eta: 0.0716 | Regression_pt: 0.3769 | Regression_sin_phi: 0.0505 | Group Train Loss Unweighted (window mean): physical_pu: 4.0705 | physical_nopu: 5.1319 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4765 | Classification_binary: 3.8369 | Regression_cos_phi: 1.9006 | Regression_energy: 2.2150 | Regression_eta: 1.2886 | Regression_pt: 2.6575 | Regression_sin_phi: 2.3956 | LR: 3.99e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.0073s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0782 | max_loss: 0.0037 | max_backward: 0.0505 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 1.0271 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4295.2 | uss_MB: 3034.9 | child_rss_MB: 11625.5 | allocated_MB: 607.6 | reserved_MB: 28576.0 | inactive_split_MB: 261.0 +rank7 [2026-10-27 16:25:44,194] INFO : Step 4000/100000 rank7 | Train Loss: 12.6160 | Train Loss Unweighted: 3.9080 | Train Loss Components: Classification: 0.0161 | Classification_binary: 3.1145 | Regression_cos_phi: 0.0484 | Regression_energy: 0.3088 | Regression_eta: 0.1117 | Regression_pt: 0.2597 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.9908 | physical_nopu: 5.0219 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4730 | Classification_binary: 3.1364 | Regression_cos_phi: 1.8273 | Regression_energy: 1.6721 | Regression_eta: 1.8537 | Regression_pt: 1.8708 | Regression_sin_phi: 2.0443 | LR: 3.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.4271s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0032 | max_forward: 0.1365 | max_loss: 0.0036 | max_backward: 0.2496 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 1.0566 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 4292.4 | uss_MB: 3043.1 | child_rss_MB: 11491.5 | allocated_MB: 617.3 | reserved_MB: 28576.0 | inactive_split_MB: 1049.1 +rank7 [2026-10-27 16:29:29,026] INFO : Step 5000/100000 rank7 | Train Loss: 13.7309 | Train Loss Unweighted: 4.7575 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.8762 | Regression_cos_phi: 0.0508 | Regression_energy: 0.3661 | Regression_eta: 0.0922 | Regression_pt: 0.3105 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.9453 | physical_nopu: 4.9561 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4129 | Classification_binary: 3.9200 | Regression_cos_phi: 1.4463 | Regression_energy: 1.8470 | Regression_eta: 2.0090 | Regression_pt: 2.1608 | Regression_sin_phi: 1.8269 | LR: 3.98e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.0871s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.0008 | max_forward: 0.0371 | max_loss: 0.0036 | max_backward: 0.0568 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 2.1174 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4241.6 | uss_MB: 3036.2 | child_rss_MB: 11146.9 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 248.7 +rank7 [2026-10-27 16:33:03,739] INFO : Step 6000/100000 rank7 | Train Loss: 15.2316 | Train Loss Unweighted: 4.9080 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.7479 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5147 | Regression_eta: 0.0679 | Regression_pt: 0.4609 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.9059 | physical_nopu: 4.9043 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5904 | Classification_binary: 3.4818 | Regression_cos_phi: 1.4720 | Regression_energy: 2.7589 | Regression_eta: 0.8636 | Regression_pt: 3.4221 | Regression_sin_phi: 2.0986 | LR: 3.97e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.0046s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0421 | max_loss: 0.0036 | max_backward: 0.0620 | max_optimizer_step: 0.1489 | max_tensorboard_logging: 2.0592 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4216.9 | uss_MB: 3036.3 | child_rss_MB: 11044.6 | allocated_MB: 610.3 | reserved_MB: 28576.0 | inactive_split_MB: 254.0 +rank7 [2026-10-27 16:36:49,903] INFO : Step 7000/100000 rank7 | Train Loss: 14.7215 | Train Loss Unweighted: 4.8917 | Train Loss Components: Classification: 0.0113 | Classification_binary: 3.8534 | Regression_cos_phi: 0.0503 | Regression_energy: 0.4508 | Regression_eta: 0.0791 | Regression_pt: 0.3921 | Regression_sin_phi: 0.0547 | Group Train Loss Unweighted (window mean): physical_pu: 3.8763 | physical_nopu: 4.8627 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2594 | Classification_binary: 3.7665 | Regression_cos_phi: 1.3560 | Regression_energy: 3.7283 | Regression_eta: 1.2023 | Regression_pt: 4.8146 | Regression_sin_phi: 1.7674 | LR: 3.96e-04 | DataLoad Time: 0.0011s | Model Forward Time: 4.5453s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0348 | max_loss: 0.0036 | max_backward: 0.0535 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 3.5409 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4194.7 | uss_MB: 3033.5 | child_rss_MB: 10983.9 | allocated_MB: 609.0 | reserved_MB: 28576.0 | inactive_split_MB: 216.2 +rank7 [2026-10-27 16:40:34,139] INFO : Step 8000/100000 rank7 | Train Loss: 14.2560 | Train Loss Unweighted: 4.8042 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.8045 | Regression_cos_phi: 0.0530 | Regression_energy: 0.4333 | Regression_eta: 0.0601 | Regression_pt: 0.3883 | Regression_sin_phi: 0.0469 | Group Train Loss Unweighted (window mean): physical_pu: 3.8518 | physical_nopu: 4.8294 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5541 | Classification_binary: 3.8583 | Regression_cos_phi: 2.2339 | Regression_energy: 2.5790 | Regression_eta: 1.0896 | Regression_pt: 3.0728 | Regression_sin_phi: 1.8662 | LR: 3.94e-04 | DataLoad Time: 0.0019s | Model Forward Time: 4.8860s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1342 | max_loss: 0.0036 | max_backward: 0.0500 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 3.8645 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4176.9 | uss_MB: 3036.4 | child_rss_MB: 10934.3 | allocated_MB: 606.6 | reserved_MB: 28576.0 | inactive_split_MB: 255.4 +rank7 [2026-10-27 16:44:13,307] INFO : Step 9000/100000 rank7 | Train Loss: 15.9274 | Train Loss Unweighted: 4.9361 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.6437 | Regression_cos_phi: 0.0472 | Regression_energy: 0.5701 | Regression_eta: 0.0733 | Regression_pt: 0.5360 | Regression_sin_phi: 0.0496 | Group Train Loss Unweighted (window mean): physical_pu: 3.8376 | physical_nopu: 4.8057 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5301 | Classification_binary: 3.6473 | Regression_cos_phi: 1.6788 | Regression_energy: 2.8484 | Regression_eta: 0.9394 | Regression_pt: 3.5121 | Regression_sin_phi: 1.5790 | LR: 3.93e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.2108s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1320 | max_loss: 0.0037 | max_backward: 0.0555 | max_optimizer_step: 0.1493 | max_tensorboard_logging: 2.0797 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 4157.5 | uss_MB: 3035.9 | child_rss_MB: 10877.2 | allocated_MB: 607.8 | reserved_MB: 28576.0 | inactive_split_MB: 321.4 +rank7 [2026-10-27 16:48:05,432] INFO : Step 10000/100000 rank7 | Train Loss: 13.0514 | Train Loss Unweighted: 4.5682 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.7678 | Regression_cos_phi: 0.0520 | Regression_energy: 0.3372 | Regression_eta: 0.0736 | Regression_pt: 0.2745 | Regression_sin_phi: 0.0521 | Group Train Loss Unweighted (window mean): physical_pu: 3.8237 | physical_nopu: 4.7797 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3911 | Classification_binary: 3.8397 | Regression_cos_phi: 1.9213 | Regression_energy: 1.8102 | Regression_eta: 1.1652 | Regression_pt: 1.9615 | Regression_sin_phi: 2.2632 | LR: 3.91e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.2567s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.0379 | max_loss: 0.0040 | max_backward: 0.0556 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 2.2516 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4149.4 | uss_MB: 3036.1 | child_rss_MB: 10872.4 | allocated_MB: 607.9 | reserved_MB: 28576.0 | inactive_split_MB: 258.7 +rank7 [2026-10-27 16:48:06,334] INFO : Running validation on rank7 +rank7 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 16:48:06,335] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 16:50:06,285] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:50:06,772] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 16:50:34,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 16:50:35,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 16:50:36,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 16:50:36,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 16:50:37,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 16:50:37,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 16:50:38,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 16:50:38,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 16:50:48,691] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-27 16:50:48,968] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:50:49,338] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 16:51:04,879] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 16:51:05,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 16:51:05,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 16:51:06,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 16:51:06,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 16:51:07,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 16:51:07,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 16:51:08,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 16:51:18,458] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 16:51:18,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 16:51:19,155] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 16:51:34,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 16:51:35,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 16:51:35,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 16:51:36,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 16:51:36,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 16:51:37,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 16:51:37,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 16:51:38,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_10000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 16:51:48,455] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 16:55:58,693] INFO : Step 11000/100000 rank7 | Train Loss: 14.4095 | Train Loss Unweighted: 4.7539 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.7591 | Regression_cos_phi: 0.0522 | Regression_energy: 0.4228 | Regression_eta: 0.0709 | Regression_pt: 0.3796 | Regression_sin_phi: 0.0518 | Group Train Loss Unweighted (window mean): physical_pu: 3.8077 | physical_nopu: 4.7444 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5079 | Classification_binary: 3.7743 | Regression_cos_phi: 1.8409 | Regression_energy: 2.0045 | Regression_eta: 1.2120 | Regression_pt: 2.3514 | Regression_sin_phi: 2.1042 | LR: 3.89e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.1998s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1127 | max_loss: 0.0074 | max_backward: 0.1410 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 1.9404 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6580.9 | uss_MB: 5462.7 | child_rss_MB: 10900.1 | allocated_MB: 610.6 | reserved_MB: 27422.0 | inactive_split_MB: 253.4 +rank7 [2026-10-27 16:59:46,314] INFO : Step 12000/100000 rank7 | Train Loss: 16.4725 | Train Loss Unweighted: 5.0863 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.5958 | Regression_cos_phi: 0.0410 | Regression_energy: 0.6842 | Regression_eta: 0.0610 | Regression_pt: 0.6484 | Regression_sin_phi: 0.0394 | Group Train Loss Unweighted (window mean): physical_pu: 3.8013 | physical_nopu: 4.7308 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5056 | Classification_binary: 3.5304 | Regression_cos_phi: 1.4180 | Regression_energy: 3.3670 | Regression_eta: 1.2620 | Regression_pt: 4.2437 | Regression_sin_phi: 1.3558 | LR: 3.87e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.5367s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1292 | max_loss: 0.0037 | max_backward: 0.1951 | max_optimizer_step: 0.2940 | max_tensorboard_logging: 2.1160 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6552.9 | uss_MB: 5462.9 | child_rss_MB: 10800.4 | allocated_MB: 609.5 | reserved_MB: 27422.0 | inactive_split_MB: 347.2 +rank7 [2026-10-27 17:03:40,557] INFO : Step 13000/100000 rank7 | Train Loss: 12.9993 | Train Loss Unweighted: 4.5752 | Train Loss Components: Classification: 0.0115 | Classification_binary: 3.7655 | Regression_cos_phi: 0.0483 | Regression_energy: 0.3375 | Regression_eta: 0.0674 | Regression_pt: 0.2918 | Regression_sin_phi: 0.0532 | Group Train Loss Unweighted (window mean): physical_pu: 3.7898 | physical_nopu: 4.7063 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3418 | Classification_binary: 3.6989 | Regression_cos_phi: 1.9234 | Regression_energy: 1.5595 | Regression_eta: 1.1139 | Regression_pt: 1.7932 | Regression_sin_phi: 2.1263 | LR: 3.85e-04 | DataLoad Time: 0.0087s | Model Forward Time: 3.0970s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.0837 | max_loss: 0.0036 | max_backward: 0.1327 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.9573 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5436.1 | uss_MB: 4383.7 | child_rss_MB: 10499.2 | allocated_MB: 607.2 | reserved_MB: 28584.0 | inactive_split_MB: 340.8 +rank7 [2026-10-27 17:07:23,686] INFO : Step 14000/100000 rank7 | Train Loss: 12.1035 | Train Loss Unweighted: 3.7686 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.0394 | Regression_cos_phi: 0.0492 | Regression_energy: 0.2767 | Regression_eta: 0.1064 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0483 | Group Train Loss Unweighted (window mean): physical_pu: 3.7803 | physical_nopu: 4.6878 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4839 | Classification_binary: 3.0533 | Regression_cos_phi: 1.8272 | Regression_energy: 1.4514 | Regression_eta: 1.6217 | Regression_pt: 1.6400 | Regression_sin_phi: 1.8318 | LR: 3.83e-04 | DataLoad Time: 0.0018s | Model Forward Time: 5.5487s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0034 | max_forward: 0.2002 | max_loss: 0.0036 | max_backward: 0.3668 | max_optimizer_step: 0.2700 | max_tensorboard_logging: 3.9098 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0824 | Memory: rss_MB: 5443.1 | uss_MB: 4393.2 | child_rss_MB: 10497.0 | allocated_MB: 618.5 | reserved_MB: 28584.0 | inactive_split_MB: 893.5 +rank7 [2026-10-27 17:11:07,917] INFO : Step 15000/100000 rank7 | Train Loss: 13.2483 | Train Loss Unweighted: 4.5172 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.6965 | Regression_cos_phi: 0.0560 | Regression_energy: 0.3441 | Regression_eta: 0.0751 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7702 | physical_nopu: 4.6706 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5101 | Classification_binary: 3.6743 | Regression_cos_phi: 2.1355 | Regression_energy: 2.0240 | Regression_eta: 0.8760 | Regression_pt: 2.4542 | Regression_sin_phi: 2.3431 | LR: 3.80e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.3452s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.0879 | max_loss: 0.0036 | max_backward: 0.0859 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 2.2540 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0190 | Memory: rss_MB: 5437.1 | uss_MB: 4385.0 | child_rss_MB: 10498.2 | allocated_MB: 605.9 | reserved_MB: 28584.0 | inactive_split_MB: 314.1 +rank7 [2026-10-27 17:14:55,829] INFO : Step 16000/100000 rank7 | Train Loss: 14.0060 | Train Loss Unweighted: 4.5724 | Train Loss Components: Classification: 0.0150 | Classification_binary: 3.5827 | Regression_cos_phi: 0.0469 | Regression_energy: 0.4276 | Regression_eta: 0.0760 | Regression_pt: 0.3725 | Regression_sin_phi: 0.0517 | Group Train Loss Unweighted (window mean): physical_pu: 3.7654 | physical_nopu: 4.6500 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4777 | Classification_binary: 3.6780 | Regression_cos_phi: 2.3550 | Regression_energy: 2.2590 | Regression_eta: 1.3599 | Regression_pt: 2.4696 | Regression_sin_phi: 2.4306 | LR: 3.78e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.4622s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0984 | max_forward: 0.1749 | max_loss: 0.0039 | max_backward: 0.0976 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 2.1733 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5435.1 | uss_MB: 4384.3 | child_rss_MB: 10495.1 | allocated_MB: 607.6 | reserved_MB: 28978.0 | inactive_split_MB: 330.4 +rank7 [2026-10-27 17:18:33,642] INFO : Step 17000/100000 rank7 | Train Loss: 14.2557 | Train Loss Unweighted: 4.6663 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6292 | Regression_cos_phi: 0.0493 | Regression_energy: 0.4540 | Regression_eta: 0.0715 | Regression_pt: 0.3998 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.7576 | physical_nopu: 4.6430 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3939 | Classification_binary: 3.6810 | Regression_cos_phi: 2.3221 | Regression_energy: 1.8313 | Regression_eta: 1.0932 | Regression_pt: 2.2832 | Regression_sin_phi: 1.8452 | LR: 3.75e-04 | DataLoad Time: 0.0029s | Model Forward Time: 3.9701s | Phase Times: max_pre_step_barrier: 0.5705 | max_batch_to_device: 0.0043 | max_forward: 0.0887 | max_loss: 0.0037 | max_backward: 0.1154 | max_optimizer_step: 0.2542 | max_tensorboard_logging: 2.1801 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5421.0 | uss_MB: 4384.2 | child_rss_MB: 10406.9 | allocated_MB: 609.7 | reserved_MB: 29500.0 | inactive_split_MB: 302.1 +rank7 [2026-10-27 17:22:18,273] INFO : Step 18000/100000 rank7 | Train Loss: 13.2896 | Train Loss Unweighted: 4.5062 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.6504 | Regression_cos_phi: 0.0492 | Regression_energy: 0.3607 | Regression_eta: 0.0732 | Regression_pt: 0.3062 | Regression_sin_phi: 0.0530 | Group Train Loss Unweighted (window mean): physical_pu: 3.7500 | physical_nopu: 4.6286 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4390 | Classification_binary: 3.6956 | Regression_cos_phi: 1.9248 | Regression_energy: 1.6543 | Regression_eta: 0.9162 | Regression_pt: 1.9443 | Regression_sin_phi: 2.9361 | LR: 3.72e-04 | DataLoad Time: 0.0020s | Model Forward Time: 10.0051s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0832 | max_loss: 0.0037 | max_backward: 0.0999 | max_optimizer_step: 0.1725 | max_tensorboard_logging: 8.8254 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 5416.7 | uss_MB: 4384.0 | child_rss_MB: 10363.7 | allocated_MB: 608.8 | reserved_MB: 29500.0 | inactive_split_MB: 196.1 +rank7 [2026-10-27 17:25:51,406] INFO : Step 19000/100000 rank7 | Train Loss: 15.2449 | Train Loss Unweighted: 4.8480 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.5669 | Regression_cos_phi: 0.0399 | Regression_energy: 0.5792 | Regression_eta: 0.0602 | Regression_pt: 0.5415 | Regression_sin_phi: 0.0443 | Group Train Loss Unweighted (window mean): physical_pu: 3.7399 | physical_nopu: 4.6217 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4471 | Classification_binary: 3.5763 | Regression_cos_phi: 1.5165 | Regression_energy: 3.2721 | Regression_eta: 1.1137 | Regression_pt: 4.0263 | Regression_sin_phi: 1.6882 | LR: 3.69e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.1643s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0008 | max_forward: 0.0818 | max_loss: 0.0762 | max_backward: 0.1759 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 1.9822 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5410.7 | uss_MB: 4384.1 | child_rss_MB: 10330.8 | allocated_MB: 607.1 | reserved_MB: 29500.0 | inactive_split_MB: 236.2 +rank7 [2026-10-27 17:29:45,026] INFO : Step 20000/100000 rank7 | Train Loss: 13.7275 | Train Loss Unweighted: 4.6495 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.7120 | Regression_cos_phi: 0.0510 | Regression_energy: 0.4061 | Regression_eta: 0.0646 | Regression_pt: 0.3531 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.7387 | physical_nopu: 4.6038 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3522 | Classification_binary: 3.8965 | Regression_cos_phi: 1.9882 | Regression_energy: 1.5035 | Regression_eta: 0.9755 | Regression_pt: 1.6150 | Regression_sin_phi: 2.0076 | LR: 3.66e-04 | DataLoad Time: 0.0027s | Model Forward Time: 5.4931s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0007 | max_forward: 0.0851 | max_loss: 0.0039 | max_backward: 0.2016 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 4.2823 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5411.1 | uss_MB: 4383.8 | child_rss_MB: 10307.1 | allocated_MB: 609.5 | reserved_MB: 29500.0 | inactive_split_MB: 252.8 +rank7 [2026-10-27 17:29:45,970] INFO : Running validation on rank7 +rank7 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 17:29:45,970] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 17:31:42,604] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 17:31:43,254] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 17:32:09,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 17:32:10,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 17:32:10,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 17:32:11,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 17:32:11,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 17:32:12,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 17:32:12,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 17:32:13,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 17:32:23,459] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-27 17:32:23,636] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 17:32:24,023] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 17:32:39,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 17:32:40,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 17:32:40,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 17:32:41,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 17:32:41,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 17:32:42,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 17:32:42,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 17:32:43,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 17:32:53,162] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-27 17:32:53,432] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 17:32:53,822] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 17:33:09,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 17:33:09,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 17:33:10,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 17:33:10,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 17:33:11,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 17:33:11,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 17:33:12,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 17:33:12,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_20000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 17:33:22,571] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 17:37:33,417] INFO : Step 21000/100000 rank7 | Train Loss: 13.1647 | Train Loss Unweighted: 4.3967 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.5106 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3742 | Regression_eta: 0.0796 | Regression_pt: 0.3229 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.7328 | physical_nopu: 4.5933 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4743 | Classification_binary: 3.4694 | Regression_cos_phi: 1.6759 | Regression_energy: 2.1507 | Regression_eta: 1.5160 | Regression_pt: 2.3460 | Regression_sin_phi: 2.2776 | LR: 3.62e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.6671s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0483 | max_forward: 0.1678 | max_loss: 0.0037 | max_backward: 0.1430 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 2.3569 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5516.5 | uss_MB: 4216.2 | child_rss_MB: 10241.6 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 238.6 +rank7 [2026-10-27 17:41:10,019] INFO : Step 22000/100000 rank7 | Train Loss: 15.3802 | Train Loss Unweighted: 4.7914 | Train Loss Components: Classification: 0.0152 | Classification_binary: 3.5311 | Regression_cos_phi: 0.0475 | Regression_energy: 0.5593 | Regression_eta: 0.0730 | Regression_pt: 0.5219 | Regression_sin_phi: 0.0434 | Group Train Loss Unweighted (window mean): physical_pu: 3.7290 | physical_nopu: 4.5858 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5246 | Classification_binary: 3.5570 | Regression_cos_phi: 1.5012 | Regression_energy: 2.7775 | Regression_eta: 1.0318 | Regression_pt: 3.4308 | Regression_sin_phi: 1.5366 | LR: 3.59e-04 | DataLoad Time: 0.0017s | Model Forward Time: 3.1743s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.0905 | max_loss: 0.0038 | max_backward: 0.1361 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 2.0169 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5517.5 | uss_MB: 4498.8 | child_rss_MB: 10247.2 | allocated_MB: 609.4 | reserved_MB: 28490.0 | inactive_split_MB: 321.0 +rank7 [2026-10-27 17:44:47,650] INFO : Step 23000/100000 rank7 | Train Loss: 12.9848 | Train Loss Unweighted: 4.5226 | Train Loss Components: Classification: 0.0099 | Classification_binary: 3.7377 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3326 | Regression_eta: 0.0739 | Regression_pt: 0.2617 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.7288 | physical_nopu: 4.5774 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3656 | Classification_binary: 3.8269 | Regression_cos_phi: 2.0977 | Regression_energy: 1.6767 | Regression_eta: 1.2510 | Regression_pt: 1.7313 | Regression_sin_phi: 2.0257 | LR: 3.55e-04 | DataLoad Time: 0.0068s | Model Forward Time: 3.1588s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0994 | max_loss: 0.0039 | max_backward: 0.1244 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.9838 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 5517.5 | uss_MB: 4498.0 | child_rss_MB: 10251.2 | allocated_MB: 608.2 | reserved_MB: 29422.0 | inactive_split_MB: 218.5 +rank7 [2026-10-27 17:48:44,022] INFO : Step 24000/100000 rank7 | Train Loss: 16.3896 | Train Loss Unweighted: 4.8949 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4510 | Regression_cos_phi: 0.0465 | Regression_energy: 0.6613 | Regression_eta: 0.0617 | Regression_pt: 0.6132 | Regression_sin_phi: 0.0453 | Group Train Loss Unweighted (window mean): physical_pu: 3.7170 | physical_nopu: 4.5767 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4675 | Classification_binary: 3.6476 | Regression_cos_phi: 1.7124 | Regression_energy: 4.0707 | Regression_eta: 1.2372 | Regression_pt: 4.8047 | Regression_sin_phi: 1.8045 | LR: 3.51e-04 | DataLoad Time: 0.0025s | Model Forward Time: 3.1026s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0398 | max_forward: 0.1314 | max_loss: 0.0037 | max_backward: 0.0994 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 1.8290 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5518.7 | uss_MB: 4498.4 | child_rss_MB: 10238.8 | allocated_MB: 606.3 | reserved_MB: 29422.0 | inactive_split_MB: 288.6 +rank7 [2026-10-27 17:52:29,098] INFO : Step 25000/100000 rank7 | Train Loss: 13.0062 | Train Loss Unweighted: 4.4048 | Train Loss Components: Classification: 0.0114 | Classification_binary: 3.6384 | Regression_cos_phi: 0.0552 | Regression_energy: 0.3153 | Regression_eta: 0.0680 | Regression_pt: 0.2574 | Regression_sin_phi: 0.0590 | Group Train Loss Unweighted (window mean): physical_pu: 3.7138 | physical_nopu: 4.5644 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3983 | Classification_binary: 3.4526 | Regression_cos_phi: 1.7530 | Regression_energy: 1.6490 | Regression_eta: 1.0007 | Regression_pt: 1.7232 | Regression_sin_phi: 2.1993 | LR: 3.47e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.5989s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0007 | max_forward: 0.1394 | max_loss: 0.0036 | max_backward: 0.2004 | max_optimizer_step: 0.1603 | max_tensorboard_logging: 1.3791 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5517.7 | uss_MB: 4498.3 | child_rss_MB: 10255.6 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 213.9 +rank7 [2026-10-27 17:56:10,619] INFO : Step 26000/100000 rank7 | Train Loss: 13.5469 | Train Loss Unweighted: 4.4523 | Train Loss Components: Classification: 0.0185 | Classification_binary: 3.5246 | Regression_cos_phi: 0.0498 | Regression_energy: 0.3935 | Regression_eta: 0.0732 | Regression_pt: 0.3461 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5624 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5055 | Classification_binary: 3.5471 | Regression_cos_phi: 1.9683 | Regression_energy: 1.9954 | Regression_eta: 1.1966 | Regression_pt: 2.4949 | Regression_sin_phi: 2.0251 | LR: 3.43e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.1213s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0838 | max_forward: 0.1679 | max_loss: 0.0039 | max_backward: 0.0991 | max_optimizer_step: 0.2674 | max_tensorboard_logging: 1.6325 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5515.7 | uss_MB: 4498.4 | child_rss_MB: 10248.9 | allocated_MB: 608.1 | reserved_MB: 29422.0 | inactive_split_MB: 201.9 +rank7 [2026-10-27 17:59:48,627] INFO : Step 27000/100000 rank7 | Train Loss: 17.1780 | Train Loss Unweighted: 5.0531 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.5146 | Regression_cos_phi: 0.0448 | Regression_energy: 0.6955 | Regression_eta: 0.0778 | Regression_pt: 0.6583 | Regression_sin_phi: 0.0464 | Group Train Loss Unweighted (window mean): physical_pu: 3.7098 | physical_nopu: 4.5579 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4723 | Classification_binary: 3.4635 | Regression_cos_phi: 1.2145 | Regression_energy: 4.0580 | Regression_eta: 1.0598 | Regression_pt: 5.1541 | Regression_sin_phi: 1.8324 | LR: 3.39e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.4475s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1019 | max_loss: 0.0489 | max_backward: 0.1596 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.2320 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5507.7 | uss_MB: 4499.5 | child_rss_MB: 10244.8 | allocated_MB: 611.7 | reserved_MB: 30248.0 | inactive_split_MB: 336.3 +rank7 [2026-10-27 18:03:26,754] INFO : Step 28000/100000 rank7 | Train Loss: 12.6721 | Train Loss Unweighted: 4.3436 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5550 | Regression_cos_phi: 0.0488 | Regression_energy: 0.3264 | Regression_eta: 0.0754 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0507 | Group Train Loss Unweighted (window mean): physical_pu: 3.7057 | physical_nopu: 4.5482 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3996 | Classification_binary: 3.7842 | Regression_cos_phi: 2.2179 | Regression_energy: 1.9404 | Regression_eta: 1.2932 | Regression_pt: 2.0106 | Regression_sin_phi: 2.2474 | LR: 3.35e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.0170s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.0898 | max_loss: 0.0057 | max_backward: 0.1164 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.8021 | max_tensorboard_flush: 0.0040 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 5489.8 | uss_MB: 4496.9 | child_rss_MB: 10097.2 | allocated_MB: 606.6 | reserved_MB: 30248.0 | inactive_split_MB: 278.7 +rank7 [2026-10-27 18:07:25,425] INFO : Step 29000/100000 rank7 | Train Loss: 14.8403 | Train Loss Unweighted: 4.5753 | Train Loss Components: Classification: 0.0171 | Classification_binary: 3.4458 | Regression_cos_phi: 0.0494 | Regression_energy: 0.4901 | Regression_eta: 0.0767 | Regression_pt: 0.4459 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.7068 | physical_nopu: 4.5439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5788 | Classification_binary: 3.5310 | Regression_cos_phi: 1.9867 | Regression_energy: 3.0144 | Regression_eta: 1.3387 | Regression_pt: 3.6029 | Regression_sin_phi: 1.8009 | LR: 3.30e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.3380s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1464 | max_loss: 0.0037 | max_backward: 0.0935 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 1.1535 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5489.8 | uss_MB: 4498.4 | child_rss_MB: 10094.9 | allocated_MB: 607.4 | reserved_MB: 30248.0 | inactive_split_MB: 313.3 +rank7 [2026-10-27 18:11:12,494] INFO : Step 30000/100000 rank7 | Train Loss: 13.2916 | Train Loss Unweighted: 4.4954 | Train Loss Components: Classification: 0.0126 | Classification_binary: 3.5989 | Regression_cos_phi: 0.0482 | Regression_energy: 0.3865 | Regression_eta: 0.0707 | Regression_pt: 0.3291 | Regression_sin_phi: 0.0494 | Group Train Loss Unweighted (window mean): physical_pu: 3.7005 | physical_nopu: 4.5378 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4349 | Classification_binary: 3.5979 | Regression_cos_phi: 2.1332 | Regression_energy: 2.8080 | Regression_eta: 1.1328 | Regression_pt: 3.0443 | Regression_sin_phi: 2.4853 | LR: 3.26e-04 | DataLoad Time: 0.0016s | Model Forward Time: 2.7231s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0007 | max_forward: 0.0805 | max_loss: 0.0038 | max_backward: 0.0926 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 1.5279 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 5479.8 | uss_MB: 4498.3 | child_rss_MB: 10016.6 | allocated_MB: 608.2 | reserved_MB: 30760.0 | inactive_split_MB: 232.9 +rank7 [2026-10-27 18:11:13,431] INFO : Running validation on rank7 +rank7 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 18:11:13,431] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 18:11:13,432] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 18:11:13,432] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 18:13:07,586] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 18:13:08,047] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 18:13:34,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 18:13:35,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 18:13:35,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 18:13:36,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 18:13:36,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 18:13:37,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 18:13:37,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 18:13:38,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 18:13:48,095] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-27 18:13:48,509] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 18:13:49,356] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 18:14:04,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 18:14:05,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 18:14:05,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 18:14:06,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 18:14:06,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 18:14:07,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 18:14:08,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 18:14:08,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 18:14:18,583] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-27 18:14:18,746] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 18:14:19,221] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 18:14:33,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 18:14:34,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 18:14:34,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 18:14:35,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 18:14:36,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 18:14:36,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 18:14:37,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 18:14:37,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_30000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 18:14:47,532] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 18:19:00,303] INFO : Step 31000/100000 rank7 | Train Loss: 12.7750 | Train Loss Unweighted: 4.3889 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5406 | Regression_cos_phi: 0.0465 | Regression_energy: 0.3622 | Regression_eta: 0.0700 | Regression_pt: 0.3101 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6964 | physical_nopu: 4.5360 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4490 | Classification_binary: 3.5331 | Regression_cos_phi: 2.0014 | Regression_energy: 2.4018 | Regression_eta: 1.4473 | Regression_pt: 2.8469 | Regression_sin_phi: 2.2037 | LR: 3.21e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.3237s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.2346 | max_loss: 0.0037 | max_backward: 0.1255 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 1.8384 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1395 | Memory: rss_MB: 6773.5 | uss_MB: 5791.8 | child_rss_MB: 10011.8 | allocated_MB: 609.3 | reserved_MB: 29142.0 | inactive_split_MB: 278.5 +rank7 [2026-10-27 18:22:44,787] INFO : Step 32000/100000 rank7 | Train Loss: 14.3617 | Train Loss Unweighted: 4.6225 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4934 | Regression_cos_phi: 0.0448 | Regression_energy: 0.5043 | Regression_eta: 0.0718 | Regression_pt: 0.4507 | Regression_sin_phi: 0.0414 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5314 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5366 | Classification_binary: 3.5092 | Regression_cos_phi: 1.5766 | Regression_energy: 2.4739 | Regression_eta: 0.9052 | Regression_pt: 3.0504 | Regression_sin_phi: 1.5080 | LR: 3.16e-04 | DataLoad Time: 0.0103s | Model Forward Time: 2.5623s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0009 | max_forward: 0.1373 | max_loss: 0.0041 | max_backward: 0.2040 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 1.2065 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5556.1 | uss_MB: 4574.8 | child_rss_MB: 10008.9 | allocated_MB: 606.1 | reserved_MB: 29142.0 | inactive_split_MB: 511.2 +rank7 [2026-10-27 18:26:33,025] INFO : Step 33000/100000 rank7 | Train Loss: 12.0000 | Train Loss Unweighted: 3.7115 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.9674 | Regression_cos_phi: 0.0474 | Regression_energy: 0.2863 | Regression_eta: 0.1056 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6960 | physical_nopu: 4.5230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4178 | Classification_binary: 2.9780 | Regression_cos_phi: 1.7653 | Regression_energy: 1.4419 | Regression_eta: 1.5066 | Regression_pt: 1.6035 | Regression_sin_phi: 2.0085 | LR: 3.12e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.5991s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0030 | max_forward: 0.1619 | max_loss: 0.0119 | max_backward: 0.2845 | max_optimizer_step: 0.1715 | max_tensorboard_logging: 1.1927 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5557.1 | uss_MB: 4556.3 | child_rss_MB: 10013.2 | allocated_MB: 623.7 | reserved_MB: 29572.0 | inactive_split_MB: 1441.5 +rank7 [2026-10-27 18:30:15,971] INFO : Step 34000/100000 rank7 | Train Loss: 15.0627 | Train Loss Unweighted: 4.6899 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4670 | Regression_cos_phi: 0.0454 | Regression_energy: 0.5438 | Regression_eta: 0.0626 | Regression_pt: 0.5082 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6889 | physical_nopu: 4.5215 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3902 | Classification_binary: 3.5815 | Regression_cos_phi: 1.9883 | Regression_energy: 2.0468 | Regression_eta: 1.1422 | Regression_pt: 2.6082 | Regression_sin_phi: 1.8891 | LR: 3.07e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.3752s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0007 | max_forward: 0.1346 | max_loss: 0.0037 | max_backward: 0.1938 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1506 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5552.6 | uss_MB: 4573.6 | child_rss_MB: 9984.9 | allocated_MB: 609.6 | reserved_MB: 29572.0 | inactive_split_MB: 230.4 +rank7 [2026-10-27 18:33:55,272] INFO : Step 35000/100000 rank7 | Train Loss: 12.2019 | Train Loss Unweighted: 4.3540 | Train Loss Components: Classification: 0.0089 | Classification_binary: 3.6426 | Regression_cos_phi: 0.0532 | Regression_energy: 0.2965 | Regression_eta: 0.0664 | Regression_pt: 0.2370 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6854 | physical_nopu: 4.5184 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2470 | Classification_binary: 3.5025 | Regression_cos_phi: 1.6266 | Regression_energy: 1.5405 | Regression_eta: 0.6804 | Regression_pt: 1.6637 | Regression_sin_phi: 1.6234 | LR: 3.02e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.6473s | Phase Times: max_pre_step_barrier: 0.0511 | max_batch_to_device: 0.0857 | max_forward: 0.1832 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 1.2656 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 5548.6 | uss_MB: 4574.4 | child_rss_MB: 9918.7 | allocated_MB: 608.2 | reserved_MB: 29572.0 | inactive_split_MB: 245.8 +rank7 [2026-10-27 18:37:33,850] INFO : Step 36000/100000 rank7 | Train Loss: 12.7130 | Train Loss Unweighted: 4.3877 | Train Loss Components: Classification: 0.0174 | Classification_binary: 3.5479 | Regression_cos_phi: 0.0433 | Regression_energy: 0.3521 | Regression_eta: 0.0636 | Regression_pt: 0.3167 | Regression_sin_phi: 0.0466 | Group Train Loss Unweighted (window mean): physical_pu: 3.6860 | physical_nopu: 4.5137 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6254 | Classification_binary: 3.3706 | Regression_cos_phi: 1.9744 | Regression_energy: 1.9563 | Regression_eta: 1.1283 | Regression_pt: 2.3819 | Regression_sin_phi: 2.0129 | LR: 2.97e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.5819s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0007 | max_forward: 0.1685 | max_loss: 0.0137 | max_backward: 0.0851 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.2906 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 5532.6 | uss_MB: 4487.8 | child_rss_MB: 9818.0 | allocated_MB: 609.0 | reserved_MB: 29572.0 | inactive_split_MB: 268.4 +rank7 [2026-10-27 18:41:12,499] INFO : Step 37000/100000 rank7 | Train Loss: 16.1273 | Train Loss Unweighted: 4.7522 | Train Loss Components: Classification: 0.0158 | Classification_binary: 3.4049 | Regression_cos_phi: 0.0495 | Regression_energy: 0.5975 | Regression_eta: 0.0881 | Regression_pt: 0.5494 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6829 | physical_nopu: 4.5107 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4693 | Classification_binary: 3.4217 | Regression_cos_phi: 1.2682 | Regression_energy: 3.8458 | Regression_eta: 0.8649 | Regression_pt: 4.8833 | Regression_sin_phi: 1.2279 | LR: 2.91e-04 | DataLoad Time: 0.0028s | Model Forward Time: 2.3164s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0884 | max_loss: 0.0036 | max_backward: 0.0963 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 1.0635 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0656 | Memory: rss_MB: 5529.2 | uss_MB: 4574.6 | child_rss_MB: 9790.1 | allocated_MB: 605.6 | reserved_MB: 29572.0 | inactive_split_MB: 233.2 +rank7 [2026-10-27 18:44:44,282] INFO : Step 38000/100000 rank7 | Train Loss: 12.5142 | Train Loss Unweighted: 4.4218 | Train Loss Components: Classification: 0.0116 | Classification_binary: 3.6290 | Regression_cos_phi: 0.0475 | Regression_energy: 0.3376 | Regression_eta: 0.0660 | Regression_pt: 0.2827 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6875 | physical_nopu: 4.5136 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3179 | Classification_binary: 3.5615 | Regression_cos_phi: 1.6461 | Regression_energy: 1.6518 | Regression_eta: 1.0268 | Regression_pt: 1.7985 | Regression_sin_phi: 1.9036 | LR: 2.86e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.4327s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.0807 | max_loss: 0.0039 | max_backward: 0.1583 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.2386 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5519.2 | uss_MB: 4574.3 | child_rss_MB: 9713.9 | allocated_MB: 607.7 | reserved_MB: 29572.0 | inactive_split_MB: 203.1 +rank7 [2026-10-27 18:48:30,202] INFO : Step 39000/100000 rank7 | Train Loss: 13.9834 | Train Loss Unweighted: 4.4509 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4391 | Regression_cos_phi: 0.0477 | Regression_energy: 0.4363 | Regression_eta: 0.0792 | Regression_pt: 0.3839 | Regression_sin_phi: 0.0489 | Group Train Loss Unweighted (window mean): physical_pu: 3.6802 | physical_nopu: 4.5058 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5398 | Classification_binary: 3.3499 | Regression_cos_phi: 1.6405 | Regression_energy: 2.5251 | Regression_eta: 1.5615 | Regression_pt: 3.0326 | Regression_sin_phi: 2.0931 | LR: 2.81e-04 | DataLoad Time: 0.0022s | Model Forward Time: 3.8724s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.0950 | max_loss: 0.0039 | max_backward: 0.1102 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 2.6970 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5506.6 | uss_MB: 4574.6 | child_rss_MB: 9623.2 | allocated_MB: 611.4 | reserved_MB: 29682.0 | inactive_split_MB: 276.6 +rank7 [2026-10-27 18:52:07,824] INFO : Step 40000/100000 rank7 | Train Loss: 15.4467 | Train Loss Unweighted: 4.8241 | Train Loss Components: Classification: 0.0121 | Classification_binary: 3.5538 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5661 | Regression_eta: 0.0744 | Regression_pt: 0.5256 | Regression_sin_phi: 0.0462 | Group Train Loss Unweighted (window mean): physical_pu: 3.6754 | physical_nopu: 4.5045 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3329 | Classification_binary: 3.5639 | Regression_cos_phi: 1.8507 | Regression_energy: 3.6107 | Regression_eta: 1.0827 | Regression_pt: 4.4140 | Regression_sin_phi: 2.0012 | LR: 2.76e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.4738s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0008 | max_forward: 0.1251 | max_loss: 0.0037 | max_backward: 0.0913 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2799 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5491.6 | uss_MB: 4574.5 | child_rss_MB: 9510.6 | allocated_MB: 607.6 | reserved_MB: 29682.0 | inactive_split_MB: 251.4 +rank7 [2026-10-27 18:52:08,683] INFO : Running validation on rank7 +rank7 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 18:52:08,683] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 18:54:11,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 18:54:12,296] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 18:54:37,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 18:54:37,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 18:54:38,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 18:54:38,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 18:54:39,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 18:54:39,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 18:54:40,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 18:54:41,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 18:54:51,019] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-27 18:54:51,320] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 18:54:51,718] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 18:55:07,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 18:55:07,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 18:55:08,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 18:55:08,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 18:55:09,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 18:55:09,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 18:55:10,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 18:55:11,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 18:55:21,015] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 18:55:21,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 18:55:21,564] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 18:55:36,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 18:55:37,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 18:55:37,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 18:55:38,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 18:55:38,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 18:55:39,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 18:55:39,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 18:55:40,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_40000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 18:55:50,404] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-27 19:00:08,492] INFO : Step 41000/100000 rank7 | Train Loss: 13.2101 | Train Loss Unweighted: 4.4466 | Train Loss Components: Classification: 0.0143 | Classification_binary: 3.5620 | Regression_cos_phi: 0.0485 | Regression_energy: 0.3777 | Regression_eta: 0.0720 | Regression_pt: 0.3238 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6774 | physical_nopu: 4.4925 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4639 | Classification_binary: 3.5305 | Regression_cos_phi: 1.7271 | Regression_energy: 1.6267 | Regression_eta: 1.2017 | Regression_pt: 1.7957 | Regression_sin_phi: 2.0056 | LR: 2.70e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.5581s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0904 | max_forward: 0.2227 | max_loss: 0.0038 | max_backward: 0.1239 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 2.1580 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6748.4 | uss_MB: 5831.9 | child_rss_MB: 9514.0 | allocated_MB: 608.3 | reserved_MB: 27230.0 | inactive_split_MB: 387.2 +rank7 [2026-10-27 19:03:45,853] INFO : Step 42000/100000 rank7 | Train Loss: 13.7061 | Train Loss Unweighted: 4.4760 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4108 | Regression_cos_phi: 0.0459 | Regression_energy: 0.4702 | Regression_eta: 0.0614 | Regression_pt: 0.4341 | Regression_sin_phi: 0.0374 | Group Train Loss Unweighted (window mean): physical_pu: 3.6711 | physical_nopu: 4.4967 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4731 | Classification_binary: 3.4938 | Regression_cos_phi: 2.6281 | Regression_energy: 1.5066 | Regression_eta: 1.3141 | Regression_pt: 1.7041 | Regression_sin_phi: 1.8781 | LR: 2.65e-04 | DataLoad Time: 0.0076s | Model Forward Time: 2.9853s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0008 | max_forward: 0.1920 | max_loss: 0.0038 | max_backward: 0.1427 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 1.6213 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6739.4 | uss_MB: 5628.8 | child_rss_MB: 9436.4 | allocated_MB: 609.4 | reserved_MB: 27632.0 | inactive_split_MB: 509.5 +rank7 [2026-10-27 19:07:23,507] INFO : Step 43000/100000 rank7 | Train Loss: 11.9489 | Train Loss Unweighted: 3.6938 | Train Loss Components: Classification: 0.0162 | Classification_binary: 2.9682 | Regression_cos_phi: 0.0472 | Regression_energy: 0.2764 | Regression_eta: 0.1089 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6702 | physical_nopu: 4.4944 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5309 | Classification_binary: 2.9389 | Regression_cos_phi: 1.7771 | Regression_energy: 1.5665 | Regression_eta: 1.7199 | Regression_pt: 1.8017 | Regression_sin_phi: 1.9916 | LR: 2.59e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.8331s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0032 | max_forward: 0.2195 | max_loss: 0.0114 | max_backward: 0.4513 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 2.0889 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0865 | Memory: rss_MB: 5560.5 | uss_MB: 3524.0 | child_rss_MB: 9348.2 | allocated_MB: 619.9 | reserved_MB: 27918.0 | inactive_split_MB: 1305.5 +rank7 [2026-10-27 19:11:14,852] INFO : Step 44000/100000 rank7 | Train Loss: 15.8790 | Train Loss Unweighted: 4.7893 | Train Loss Components: Classification: 0.0180 | Classification_binary: 3.4610 | Regression_cos_phi: 0.0438 | Regression_energy: 0.5944 | Regression_eta: 0.0747 | Regression_pt: 0.5488 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6681 | physical_nopu: 4.4856 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5269 | Classification_binary: 3.5071 | Regression_cos_phi: 1.3884 | Regression_energy: 3.4213 | Regression_eta: 0.9628 | Regression_pt: 4.3075 | Regression_sin_phi: 1.5119 | LR: 2.54e-04 | DataLoad Time: 0.0026s | Model Forward Time: 3.1648s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0009 | max_forward: 0.0980 | max_loss: 0.0531 | max_backward: 0.1371 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.9403 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5553.5 | uss_MB: 4666.8 | child_rss_MB: 9287.7 | allocated_MB: 606.4 | reserved_MB: 31154.0 | inactive_split_MB: 295.0 +rank7 [2026-10-27 19:15:01,382] INFO : Step 45000/100000 rank7 | Train Loss: 13.8786 | Train Loss Unweighted: 4.5241 | Train Loss Components: Classification: 0.0100 | Classification_binary: 3.5822 | Regression_cos_phi: 0.0536 | Regression_energy: 0.4068 | Regression_eta: 0.0847 | Regression_pt: 0.3344 | Regression_sin_phi: 0.0523 | Group Train Loss Unweighted (window mean): physical_pu: 3.6667 | physical_nopu: 4.4836 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4302 | Classification_binary: 3.7091 | Regression_cos_phi: 2.1221 | Regression_energy: 3.8413 | Regression_eta: 1.2193 | Regression_pt: 4.5347 | Regression_sin_phi: 2.2638 | LR: 2.48e-04 | DataLoad Time: 0.0014s | Model Forward Time: 4.6043s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0009 | max_forward: 0.0951 | max_loss: 0.0815 | max_backward: 0.1837 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 3.3466 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 5551.4 | uss_MB: 4666.0 | child_rss_MB: 9284.1 | allocated_MB: 608.8 | reserved_MB: 31154.0 | inactive_split_MB: 289.8 +rank7 [2026-10-27 19:18:42,715] INFO : Step 46000/100000 rank7 | Train Loss: 13.1587 | Train Loss Unweighted: 4.3779 | Train Loss Components: Classification: 0.0169 | Classification_binary: 3.4817 | Regression_cos_phi: 0.0459 | Regression_energy: 0.3805 | Regression_eta: 0.0668 | Regression_pt: 0.3375 | Regression_sin_phi: 0.0486 | Group Train Loss Unweighted (window mean): physical_pu: 3.6620 | physical_nopu: 4.4831 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5620 | Classification_binary: 3.4426 | Regression_cos_phi: 1.5963 | Regression_energy: 1.9981 | Regression_eta: 0.8484 | Regression_pt: 2.3344 | Regression_sin_phi: 1.9032 | LR: 2.43e-04 | DataLoad Time: 0.0015s | Model Forward Time: 3.3334s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0924 | max_forward: 0.1910 | max_loss: 0.0039 | max_backward: 0.1028 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 2.0151 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5545.4 | uss_MB: 4666.2 | child_rss_MB: 9248.1 | allocated_MB: 609.2 | reserved_MB: 31154.0 | inactive_split_MB: 277.4 +rank7 [2026-10-27 19:22:31,522] INFO : Step 47000/100000 rank7 | Train Loss: 15.2118 | Train Loss Unweighted: 4.6831 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3624 | Regression_cos_phi: 0.0409 | Regression_energy: 0.5987 | Regression_eta: 0.0659 | Regression_pt: 0.5598 | Regression_sin_phi: 0.0405 | Group Train Loss Unweighted (window mean): physical_pu: 3.6609 | physical_nopu: 4.4795 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5724 | Classification_binary: 3.2751 | Regression_cos_phi: 1.5291 | Regression_energy: 3.3225 | Regression_eta: 0.7870 | Regression_pt: 4.2364 | Regression_sin_phi: 1.9331 | LR: 2.37e-04 | DataLoad Time: 0.0018s | Model Forward Time: 3.2745s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0777 | max_forward: 0.1864 | max_loss: 0.0077 | max_backward: 0.1105 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 1.8392 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5533.2 | uss_MB: 4665.8 | child_rss_MB: 9129.7 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 305.0 +rank7 [2026-10-27 19:26:21,057] INFO : Step 48000/100000 rank7 | Train Loss: 12.0455 | Train Loss Unweighted: 4.3014 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.6150 | Regression_cos_phi: 0.0526 | Regression_energy: 0.2805 | Regression_eta: 0.0652 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0503 | Group Train Loss Unweighted (window mean): physical_pu: 3.6617 | physical_nopu: 4.4726 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2977 | Classification_binary: 3.4944 | Regression_cos_phi: 2.1706 | Regression_energy: 1.2583 | Regression_eta: 0.9721 | Regression_pt: 1.3986 | Regression_sin_phi: 2.1381 | LR: 2.31e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.3650s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0531 | max_forward: 0.1607 | max_loss: 0.0044 | max_backward: 0.0997 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 2.0345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5534.2 | uss_MB: 4664.3 | child_rss_MB: 9129.2 | allocated_MB: 606.8 | reserved_MB: 31154.0 | inactive_split_MB: 262.3 +rank7 [2026-10-27 19:30:06,213] INFO : Step 49000/100000 rank7 | Train Loss: 13.3973 | Train Loss Unweighted: 4.4035 | Train Loss Components: Classification: 0.0168 | Classification_binary: 3.5033 | Regression_cos_phi: 0.0506 | Regression_energy: 0.3797 | Regression_eta: 0.0752 | Regression_pt: 0.3299 | Regression_sin_phi: 0.0480 | Group Train Loss Unweighted (window mean): physical_pu: 3.6566 | physical_nopu: 4.4735 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6574 | Classification_binary: 3.3935 | Regression_cos_phi: 1.6000 | Regression_energy: 1.7236 | Regression_eta: 1.0789 | Regression_pt: 2.1258 | Regression_sin_phi: 1.8225 | LR: 2.26e-04 | DataLoad Time: 0.0023s | Model Forward Time: 4.5484s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0220 | max_forward: 0.1075 | max_loss: 0.0037 | max_backward: 0.1020 | max_optimizer_step: 0.2722 | max_tensorboard_logging: 3.2606 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5532.9 | uss_MB: 4389.0 | child_rss_MB: 9133.5 | allocated_MB: 606.5 | reserved_MB: 31154.0 | inactive_split_MB: 312.3 +rank7 [2026-10-27 19:33:47,826] INFO : Step 50000/100000 rank7 | Train Loss: 14.2027 | Train Loss Unweighted: 4.5299 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4272 | Regression_cos_phi: 0.0432 | Regression_energy: 0.4877 | Regression_eta: 0.0670 | Regression_pt: 0.4429 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6600 | physical_nopu: 4.4710 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4647 | Classification_binary: 3.5404 | Regression_cos_phi: 1.5399 | Regression_energy: 3.0491 | Regression_eta: 1.2134 | Regression_pt: 3.3770 | Regression_sin_phi: 2.1307 | LR: 2.20e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.0501s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0009 | max_forward: 0.0964 | max_loss: 0.0590 | max_backward: 0.1520 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.8031 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 5519.9 | uss_MB: 4665.8 | child_rss_MB: 9015.1 | allocated_MB: 607.2 | reserved_MB: 31154.0 | inactive_split_MB: 255.6 +rank7 [2026-10-27 19:33:48,728] INFO : Running validation on rank7 +rank7 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 19:33:48,729] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 19:35:46,023] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 19:35:46,495] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 19:36:12,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 19:36:12,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 19:36:13,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 19:36:13,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 19:36:14,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 19:36:14,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 19:36:15,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 19:36:15,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 19:36:25,725] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-27 19:36:25,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 19:36:26,397] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 19:36:41,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 19:36:42,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 19:36:42,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 19:36:43,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 19:36:43,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 19:36:44,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 19:36:44,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 19:36:45,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 19:36:55,111] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 19:36:55,547] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 19:36:55,958] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 19:37:11,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 19:37:11,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 19:37:12,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 19:37:12,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 19:37:13,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 19:37:13,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 19:37:14,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 19:37:14,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_50000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 19:37:24,755] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 19:41:35,792] INFO : Step 51000/100000 rank7 | Train Loss: 13.0997 | Train Loss Unweighted: 4.3733 | Train Loss Components: Classification: 0.0129 | Classification_binary: 3.5282 | Regression_cos_phi: 0.0525 | Regression_energy: 0.3598 | Regression_eta: 0.0785 | Regression_pt: 0.2920 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6545 | physical_nopu: 4.4631 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3402 | Classification_binary: 3.5584 | Regression_cos_phi: 1.9665 | Regression_energy: 1.8257 | Regression_eta: 1.0761 | Regression_pt: 2.0772 | Regression_sin_phi: 2.0347 | LR: 2.14e-04 | DataLoad Time: 0.0112s | Model Forward Time: 3.1044s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0008 | max_forward: 0.1313 | max_loss: 0.0271 | max_backward: 0.1689 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.7590 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6858.3 | uss_MB: 6017.9 | child_rss_MB: 8958.7 | allocated_MB: 607.6 | reserved_MB: 29536.0 | inactive_split_MB: 388.7 +rank7 [2026-10-27 19:45:15,993] INFO : Step 52000/100000 rank7 | Train Loss: 11.7496 | Train Loss Unweighted: 3.6574 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9475 | Regression_cos_phi: 0.0469 | Regression_energy: 0.2717 | Regression_eta: 0.1056 | Regression_pt: 0.2231 | Regression_sin_phi: 0.0473 | Group Train Loss Unweighted (window mean): physical_pu: 3.6567 | physical_nopu: 4.4685 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4696 | Classification_binary: 2.8851 | Regression_cos_phi: 1.7534 | Regression_energy: 1.3894 | Regression_eta: 1.5406 | Regression_pt: 1.5508 | Regression_sin_phi: 1.8594 | LR: 2.09e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.6141s | Phase Times: max_pre_step_barrier: 0.0006 | max_batch_to_device: 0.1059 | max_forward: 0.2956 | max_loss: 0.0038 | max_backward: 0.4187 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 1.8177 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6858.3 | uss_MB: 6026.1 | child_rss_MB: 8942.2 | allocated_MB: 619.9 | reserved_MB: 29990.0 | inactive_split_MB: 1078.6 +rank7 [2026-10-27 19:48:51,528] INFO : Step 53000/100000 rank7 | Train Loss: 12.5294 | Train Loss Unweighted: 4.3083 | Train Loss Components: Classification: 0.0122 | Classification_binary: 3.5302 | Regression_cos_phi: 0.0494 | Regression_energy: 0.3242 | Regression_eta: 0.0722 | Regression_pt: 0.2705 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6530 | physical_nopu: 4.4629 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3032 | Classification_binary: 3.5808 | Regression_cos_phi: 1.5110 | Regression_energy: 2.0940 | Regression_eta: 0.9069 | Regression_pt: 2.3499 | Regression_sin_phi: 2.1076 | LR: 2.03e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.5891s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0009 | max_forward: 0.1740 | max_loss: 0.0036 | max_backward: 0.1135 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 1.3124 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 5606.5 | uss_MB: 4766.5 | child_rss_MB: 8943.1 | allocated_MB: 608.5 | reserved_MB: 29990.0 | inactive_split_MB: 246.4 +rank7 [2026-10-27 19:52:28,508] INFO : Step 54000/100000 rank7 | Train Loss: 13.6142 | Train Loss Unweighted: 4.3355 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.3742 | Regression_cos_phi: 0.0499 | Regression_energy: 0.4083 | Regression_eta: 0.0736 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6492 | physical_nopu: 4.4652 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5120 | Classification_binary: 3.4254 | Regression_cos_phi: 1.8623 | Regression_energy: 1.9998 | Regression_eta: 1.6293 | Regression_pt: 2.4712 | Regression_sin_phi: 2.0359 | LR: 1.97e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.4440s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1029 | max_loss: 0.0156 | max_backward: 0.1263 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 1.2459 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 5600.5 | uss_MB: 4766.6 | child_rss_MB: 8886.2 | allocated_MB: 608.0 | reserved_MB: 29990.0 | inactive_split_MB: 249.1 +rank7 [2026-10-27 19:56:02,481] INFO : Step 55000/100000 rank7 | Train Loss: 14.4463 | Train Loss Unweighted: 4.6356 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.5250 | Regression_cos_phi: 0.0497 | Regression_energy: 0.4959 | Regression_eta: 0.0660 | Regression_pt: 0.4403 | Regression_sin_phi: 0.0455 | Group Train Loss Unweighted (window mean): physical_pu: 3.6517 | physical_nopu: 4.4554 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4231 | Classification_binary: 3.5902 | Regression_cos_phi: 1.6010 | Regression_energy: 1.8950 | Regression_eta: 1.0277 | Regression_pt: 2.0783 | Regression_sin_phi: 1.8387 | LR: 1.92e-04 | DataLoad Time: 0.0103s | Model Forward Time: 2.9721s | Phase Times: max_pre_step_barrier: 0.0108 | max_batch_to_device: 0.0008 | max_forward: 0.0923 | max_loss: 0.0219 | max_backward: 0.1306 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 1.7796 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5599.8 | uss_MB: 4765.5 | child_rss_MB: 8883.4 | allocated_MB: 610.0 | reserved_MB: 29990.0 | inactive_split_MB: 274.0 +rank7 [2026-10-27 19:59:40,797] INFO : Step 56000/100000 rank7 | Train Loss: 12.4091 | Train Loss Unweighted: 4.2572 | Train Loss Components: Classification: 0.0135 | Classification_binary: 3.4523 | Regression_cos_phi: 0.0441 | Regression_energy: 0.3421 | Regression_eta: 0.0664 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0487 | Group Train Loss Unweighted (window mean): physical_pu: 3.6493 | physical_nopu: 4.4547 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5607 | Classification_binary: 3.3802 | Regression_cos_phi: 1.8775 | Regression_energy: 2.0248 | Regression_eta: 0.9592 | Regression_pt: 2.3690 | Regression_sin_phi: 1.8078 | LR: 1.86e-04 | DataLoad Time: 0.0016s | Model Forward Time: 3.0682s | Phase Times: max_pre_step_barrier: 0.3290 | max_batch_to_device: 0.0902 | max_forward: 0.1771 | max_loss: 0.0038 | max_backward: 0.1058 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 1.4273 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5599.8 | uss_MB: 4766.9 | child_rss_MB: 8880.9 | allocated_MB: 608.1 | reserved_MB: 29990.0 | inactive_split_MB: 253.9 +rank7 [2026-10-27 20:03:29,496] INFO : Step 57000/100000 rank7 | Train Loss: 14.9087 | Train Loss Unweighted: 4.5957 | Train Loss Components: Classification: 0.0165 | Classification_binary: 3.4371 | Regression_cos_phi: 0.0471 | Regression_energy: 0.5028 | Regression_eta: 0.0770 | Regression_pt: 0.4658 | Regression_sin_phi: 0.0495 | Group Train Loss Unweighted (window mean): physical_pu: 3.6475 | physical_nopu: 4.4509 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3929 | Classification_binary: 3.5541 | Regression_cos_phi: 1.7080 | Regression_energy: 2.6018 | Regression_eta: 1.0187 | Regression_pt: 3.1635 | Regression_sin_phi: 1.9532 | LR: 1.81e-04 | DataLoad Time: 0.0029s | Model Forward Time: 2.4955s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0008 | max_forward: 0.1036 | max_loss: 0.0876 | max_backward: 0.2130 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 1.1951 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5597.8 | uss_MB: 4497.3 | child_rss_MB: 8866.5 | allocated_MB: 608.1 | reserved_MB: 30358.0 | inactive_split_MB: 283.9 +rank7 [2026-10-27 20:07:08,735] INFO : Step 58000/100000 rank7 | Train Loss: 12.4468 | Train Loss Unweighted: 4.2826 | Train Loss Components: Classification: 0.0105 | Classification_binary: 3.5299 | Regression_cos_phi: 0.0545 | Regression_energy: 0.3106 | Regression_eta: 0.0620 | Regression_pt: 0.2643 | Regression_sin_phi: 0.0509 | Group Train Loss Unweighted (window mean): physical_pu: 3.6440 | physical_nopu: 4.4492 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2718 | Classification_binary: 3.5150 | Regression_cos_phi: 2.1772 | Regression_energy: 1.2629 | Regression_eta: 0.7884 | Regression_pt: 1.3750 | Regression_sin_phi: 2.1420 | LR: 1.75e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.5425s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.1868 | max_loss: 0.0037 | max_backward: 0.1111 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2860 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5587.6 | uss_MB: 3611.2 | child_rss_MB: 8803.1 | allocated_MB: 608.2 | reserved_MB: 30358.0 | inactive_split_MB: 337.0 +rank7 [2026-10-27 20:10:52,442] INFO : Step 59000/100000 rank7 | Train Loss: 13.1870 | Train Loss Unweighted: 4.3659 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4760 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3771 | Regression_eta: 0.0773 | Regression_pt: 0.3247 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6451 | physical_nopu: 4.4440 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5401 | Classification_binary: 3.5829 | Regression_cos_phi: 1.8805 | Regression_energy: 2.4926 | Regression_eta: 1.2776 | Regression_pt: 2.8137 | Regression_sin_phi: 2.5022 | LR: 1.70e-04 | DataLoad Time: 0.0017s | Model Forward Time: 2.4661s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.1005 | max_loss: 0.0036 | max_backward: 0.1062 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 1.2390 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 5582.6 | uss_MB: 4766.7 | child_rss_MB: 8755.9 | allocated_MB: 610.1 | reserved_MB: 31234.0 | inactive_split_MB: 273.9 +rank7 [2026-10-27 20:14:34,818] INFO : Step 60000/100000 rank7 | Train Loss: 15.4000 | Train Loss Unweighted: 4.6905 | Train Loss Components: Classification: 0.0154 | Classification_binary: 3.3905 | Regression_cos_phi: 0.0456 | Regression_energy: 0.5805 | Regression_eta: 0.0703 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0431 | Group Train Loss Unweighted (window mean): physical_pu: 3.6406 | physical_nopu: 4.4523 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.6260 | Classification_binary: 3.2432 | Regression_cos_phi: 1.1741 | Regression_energy: 3.2540 | Regression_eta: 0.7607 | Regression_pt: 4.1190 | Regression_sin_phi: 1.2317 | LR: 1.64e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.5446s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0757 | max_forward: 0.1624 | max_loss: 0.0040 | max_backward: 0.1315 | max_optimizer_step: 0.2155 | max_tensorboard_logging: 1.2531 | max_tensorboard_flush: 0.0020 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5582.8 | uss_MB: 4766.6 | child_rss_MB: 8760.5 | allocated_MB: 611.6 | reserved_MB: 31234.0 | inactive_split_MB: 318.4 +rank7 [2026-10-27 20:14:35,735] INFO : Running validation on rank7 +rank7 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 20:14:35,735] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 20:14:35,736] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 20:14:35,736] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 20:16:30,818] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 20:16:31,258] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 20:16:55,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 20:16:56,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 20:16:56,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 20:16:57,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 20:16:57,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 20:16:58,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 20:16:58,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 20:16:59,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 20:17:09,164] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 20:17:09,354] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 20:17:09,707] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 20:17:25,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 20:17:25,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 20:17:26,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 20:17:26,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 20:17:27,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 20:17:27,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 20:17:28,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 20:17:28,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 20:17:38,713] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 20:17:39,022] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 20:17:39,450] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 20:17:54,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 20:17:55,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 20:17:55,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 20:17:56,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 20:17:56,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 20:17:57,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 20:17:57,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 20:17:58,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_60000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 20:18:08,384] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 20:22:28,767] INFO : Step 61000/100000 rank7 | Train Loss: 12.1474 | Train Loss Unweighted: 4.3289 | Train Loss Components: Classification: 0.0110 | Classification_binary: 3.6236 | Regression_cos_phi: 0.0523 | Regression_energy: 0.2909 | Regression_eta: 0.0654 | Regression_pt: 0.2368 | Regression_sin_phi: 0.0490 | Group Train Loss Unweighted (window mean): physical_pu: 3.6358 | physical_nopu: 4.4439 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3865 | Classification_binary: 3.4827 | Regression_cos_phi: 1.8641 | Regression_energy: 1.3865 | Regression_eta: 0.8986 | Regression_pt: 1.5577 | Regression_sin_phi: 1.6704 | LR: 1.59e-04 | DataLoad Time: 0.0090s | Model Forward Time: 3.4715s | Phase Times: max_pre_step_barrier: 0.0115 | max_batch_to_device: 0.0009 | max_forward: 0.2092 | max_loss: 0.0037 | max_backward: 0.1501 | max_optimizer_step: 0.2965 | max_tensorboard_logging: 1.8725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6817.2 | uss_MB: 6009.9 | child_rss_MB: 8755.0 | allocated_MB: 607.8 | reserved_MB: 28822.0 | inactive_split_MB: 222.9 +rank7 [2026-10-27 20:26:14,158] INFO : Step 62000/100000 rank7 | Train Loss: 11.7977 | Train Loss Unweighted: 3.6294 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.9084 | Regression_cos_phi: 0.0481 | Regression_energy: 0.2778 | Regression_eta: 0.1039 | Regression_pt: 0.2286 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6417 | physical_nopu: 4.4459 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4697 | Classification_binary: 2.9465 | Regression_cos_phi: 1.8258 | Regression_energy: 1.4650 | Regression_eta: 1.5256 | Regression_pt: 1.5990 | Regression_sin_phi: 1.8989 | LR: 1.54e-04 | DataLoad Time: 0.0026s | Model Forward Time: 2.7423s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0033 | max_forward: 0.2219 | max_loss: 0.0037 | max_backward: 0.2965 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1292 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1278 | Memory: rss_MB: 6812.4 | uss_MB: 6017.7 | child_rss_MB: 8760.3 | allocated_MB: 618.8 | reserved_MB: 28822.0 | inactive_split_MB: 888.0 +rank7 [2026-10-27 20:30:05,877] INFO : Step 63000/100000 rank7 | Train Loss: 12.5071 | Train Loss Unweighted: 4.3613 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5997 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3200 | Regression_eta: 0.0719 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6399 | physical_nopu: 4.4447 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4132 | Classification_binary: 3.6192 | Regression_cos_phi: 1.6750 | Regression_energy: 1.2296 | Regression_eta: 0.8316 | Regression_pt: 1.3753 | Regression_sin_phi: 1.6167 | LR: 1.49e-04 | DataLoad Time: 0.0012s | Model Forward Time: 11.0669s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0008 | max_forward: 0.1169 | max_loss: 0.0157 | max_backward: 0.2882 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 9.0944 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6806.2 | uss_MB: 4694.0 | child_rss_MB: 8747.4 | allocated_MB: 610.4 | reserved_MB: 28822.0 | inactive_split_MB: 261.6 +rank7 [2026-10-27 20:33:53,443] INFO : Step 64000/100000 rank7 | Train Loss: 14.0388 | Train Loss Unweighted: 4.5035 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.4200 | Regression_cos_phi: 0.0430 | Regression_energy: 0.4752 | Regression_eta: 0.0663 | Regression_pt: 0.4376 | Regression_sin_phi: 0.0442 | Group Train Loss Unweighted (window mean): physical_pu: 3.6357 | physical_nopu: 4.4395 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5202 | Classification_binary: 3.3319 | Regression_cos_phi: 1.5804 | Regression_energy: 3.2645 | Regression_eta: 1.0723 | Regression_pt: 4.1763 | Regression_sin_phi: 1.7324 | LR: 1.43e-04 | DataLoad Time: 0.0024s | Model Forward Time: 6.4866s | Phase Times: max_pre_step_barrier: 0.1731 | max_batch_to_device: 0.0118 | max_forward: 0.1219 | max_loss: 0.0038 | max_backward: 0.1121 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 5.0441 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5600.0 | uss_MB: 4820.3 | child_rss_MB: 8744.3 | allocated_MB: 611.1 | reserved_MB: 29052.0 | inactive_split_MB: 372.9 +rank7 [2026-10-27 20:37:48,515] INFO : Step 65000/100000 rank7 | Train Loss: 16.3374 | Train Loss Unweighted: 4.8919 | Train Loss Components: Classification: 0.0142 | Classification_binary: 3.4871 | Regression_cos_phi: 0.0500 | Regression_energy: 0.6360 | Regression_eta: 0.0628 | Regression_pt: 0.5943 | Regression_sin_phi: 0.0475 | Group Train Loss Unweighted (window mean): physical_pu: 3.6364 | physical_nopu: 4.4335 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4340 | Classification_binary: 3.2272 | Regression_cos_phi: 1.4556 | Regression_energy: 2.3453 | Regression_eta: 0.7518 | Regression_pt: 2.8096 | Regression_sin_phi: 1.4004 | LR: 1.38e-04 | DataLoad Time: 0.0020s | Model Forward Time: 3.4884s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0009 | max_forward: 0.2167 | max_loss: 0.0058 | max_backward: 0.2407 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 1.9060 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5594.0 | uss_MB: 4820.8 | child_rss_MB: 8750.8 | allocated_MB: 609.7 | reserved_MB: 29052.0 | inactive_split_MB: 236.9 +rank7 [2026-10-27 20:41:37,390] INFO : Step 66000/100000 rank7 | Train Loss: 12.8848 | Train Loss Unweighted: 4.2935 | Train Loss Components: Classification: 0.0146 | Classification_binary: 3.4715 | Regression_cos_phi: 0.0516 | Regression_energy: 0.3451 | Regression_eta: 0.0693 | Regression_pt: 0.2907 | Regression_sin_phi: 0.0506 | Group Train Loss Unweighted (window mean): physical_pu: 3.6349 | physical_nopu: 4.4348 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3966 | Classification_binary: 3.5060 | Regression_cos_phi: 2.1685 | Regression_energy: 1.7226 | Regression_eta: 1.2243 | Regression_pt: 1.8723 | Regression_sin_phi: 2.3096 | LR: 1.33e-04 | DataLoad Time: 0.0021s | Model Forward Time: 3.1921s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0008 | max_forward: 0.0854 | max_loss: 0.1083 | max_backward: 0.2178 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 1.9218 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5582.1 | uss_MB: 4820.7 | child_rss_MB: 8750.8 | allocated_MB: 611.4 | reserved_MB: 29052.0 | inactive_split_MB: 254.2 +rank7 [2026-10-27 20:45:24,537] INFO : Step 67000/100000 rank7 | Train Loss: 14.2125 | Train Loss Unweighted: 4.5026 | Train Loss Components: Classification: 0.0164 | Classification_binary: 3.4493 | Regression_cos_phi: 0.0506 | Regression_energy: 0.4566 | Regression_eta: 0.0671 | Regression_pt: 0.4150 | Regression_sin_phi: 0.0476 | Group Train Loss Unweighted (window mean): physical_pu: 3.6352 | physical_nopu: 4.4350 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4745 | Classification_binary: 3.4345 | Regression_cos_phi: 1.8139 | Regression_energy: 2.9332 | Regression_eta: 1.1119 | Regression_pt: 3.5816 | Regression_sin_phi: 1.7198 | LR: 1.28e-04 | DataLoad Time: 0.0030s | Model Forward Time: 3.0114s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0009 | max_forward: 0.0901 | max_loss: 0.0487 | max_backward: 0.1449 | max_optimizer_step: 0.1766 | max_tensorboard_logging: 1.8090 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 5575.1 | uss_MB: 4820.8 | child_rss_MB: 8750.2 | allocated_MB: 607.1 | reserved_MB: 29260.0 | inactive_split_MB: 246.6 +rank7 [2026-10-27 20:49:05,126] INFO : Step 68000/100000 rank7 | Train Loss: 12.0487 | Train Loss Unweighted: 4.2633 | Train Loss Components: Classification: 0.0107 | Classification_binary: 3.5135 | Regression_cos_phi: 0.0466 | Regression_energy: 0.3165 | Regression_eta: 0.0645 | Regression_pt: 0.2647 | Regression_sin_phi: 0.0467 | Group Train Loss Unweighted (window mean): physical_pu: 3.6341 | physical_nopu: 4.4387 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3835 | Classification_binary: 3.5131 | Regression_cos_phi: 1.5668 | Regression_energy: 2.1304 | Regression_eta: 0.6662 | Regression_pt: 2.5133 | Regression_sin_phi: 1.8374 | LR: 1.24e-04 | DataLoad Time: 0.0026s | Model Forward Time: 2.8889s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0010 | max_forward: 0.0879 | max_loss: 0.0120 | max_backward: 0.1069 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 1.7176 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5570.9 | uss_MB: 4820.5 | child_rss_MB: 8753.3 | allocated_MB: 609.5 | reserved_MB: 29260.0 | inactive_split_MB: 225.9 +rank7 [2026-10-27 20:52:47,283] INFO : Step 69000/100000 rank7 | Train Loss: 13.1377 | Train Loss Unweighted: 4.3133 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.4496 | Regression_cos_phi: 0.0450 | Regression_energy: 0.3632 | Regression_eta: 0.0778 | Regression_pt: 0.3079 | Regression_sin_phi: 0.0541 | Group Train Loss Unweighted (window mean): physical_pu: 3.6314 | physical_nopu: 4.4321 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3713 | Classification_binary: 3.3395 | Regression_cos_phi: 1.6873 | Regression_energy: 2.0165 | Regression_eta: 1.1728 | Regression_pt: 2.3743 | Regression_sin_phi: 2.3612 | LR: 1.19e-04 | DataLoad Time: 0.0012s | Model Forward Time: 4.9130s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0007 | max_forward: 0.1207 | max_loss: 0.0509 | max_backward: 0.2012 | max_optimizer_step: 0.1818 | max_tensorboard_logging: 3.5532 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 5571.9 | uss_MB: 4820.8 | child_rss_MB: 8755.8 | allocated_MB: 608.0 | reserved_MB: 29260.0 | inactive_split_MB: 256.4 +rank7 [2026-10-27 20:56:27,513] INFO : Step 70000/100000 rank7 | Train Loss: 15.8477 | Train Loss Unweighted: 4.7317 | Train Loss Components: Classification: 0.0163 | Classification_binary: 3.3873 | Regression_cos_phi: 0.0498 | Regression_energy: 0.6015 | Regression_eta: 0.0629 | Regression_pt: 0.5683 | Regression_sin_phi: 0.0456 | Group Train Loss Unweighted (window mean): physical_pu: 3.6315 | physical_nopu: 4.4280 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4843 | Classification_binary: 3.4174 | Regression_cos_phi: 2.0684 | Regression_energy: 2.7902 | Regression_eta: 0.9874 | Regression_pt: 3.3171 | Regression_sin_phi: 2.1152 | LR: 1.14e-04 | DataLoad Time: 0.0068s | Model Forward Time: 2.9283s | Phase Times: max_pre_step_barrier: 0.0198 | max_batch_to_device: 0.0007 | max_forward: 0.1178 | max_loss: 0.0036 | max_backward: 0.1081 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.7610 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5571.5 | uss_MB: 4819.7 | child_rss_MB: 8751.0 | allocated_MB: 610.3 | reserved_MB: 30150.0 | inactive_split_MB: 260.3 +rank7 [2026-10-27 20:56:28,457] INFO : Running validation on rank7 +rank7 [2026-10-27 20:56:28,457] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 20:56:28,458] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 20:58:25,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 20:58:26,362] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 20:58:51,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 20:58:52,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 20:58:52,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 20:58:53,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 20:58:54,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 20:58:54,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 20:58:55,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 20:58:55,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 20:59:05,685] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-27 20:59:05,946] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 20:59:06,379] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 20:59:21,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 20:59:22,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 20:59:22,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 20:59:23,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 20:59:23,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 20:59:24,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 20:59:24,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 20:59:25,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 20:59:35,439] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-27 20:59:35,863] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 20:59:36,280] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 20:59:51,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 20:59:52,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 20:59:52,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 20:59:53,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 20:59:53,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 20:59:53,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 20:59:54,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 20:59:54,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_70000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 21:00:05,000] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 21:04:25,115] INFO : Step 71000/100000 rank7 | Train Loss: 12.0110 | Train Loss Unweighted: 3.6539 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9046 | Regression_cos_phi: 0.0476 | Regression_energy: 0.2881 | Regression_eta: 0.1063 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0477 | Group Train Loss Unweighted (window mean): physical_pu: 3.6306 | physical_nopu: 4.4256 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4838 | Classification_binary: 2.8856 | Regression_cos_phi: 1.7702 | Regression_energy: 1.6145 | Regression_eta: 1.5966 | Regression_pt: 1.8975 | Regression_sin_phi: 1.9466 | LR: 1.10e-04 | DataLoad Time: 0.0019s | Model Forward Time: 3.4583s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0035 | max_forward: 0.1952 | max_loss: 0.0038 | max_backward: 0.3195 | max_optimizer_step: 0.3092 | max_tensorboard_logging: 1.7411 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6798.7 | uss_MB: 6058.3 | child_rss_MB: 8722.6 | allocated_MB: 623.5 | reserved_MB: 27422.0 | inactive_split_MB: 864.2 +rank7 [2026-10-27 21:08:01,993] INFO : Step 72000/100000 rank7 | Train Loss: 15.2878 | Train Loss Unweighted: 4.6447 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.3643 | Regression_cos_phi: 0.0460 | Regression_energy: 0.5735 | Regression_eta: 0.0647 | Regression_pt: 0.5353 | Regression_sin_phi: 0.0449 | Group Train Loss Unweighted (window mean): physical_pu: 3.6275 | physical_nopu: 4.4239 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4717 | Classification_binary: 3.4055 | Regression_cos_phi: 1.3615 | Regression_energy: 2.4604 | Regression_eta: 0.8092 | Regression_pt: 3.0798 | Regression_sin_phi: 1.3771 | LR: 1.05e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.5453s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0232 | max_forward: 0.1174 | max_loss: 0.0036 | max_backward: 0.0921 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 1.2606 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5641.1 | uss_MB: 4896.7 | child_rss_MB: 8682.2 | allocated_MB: 607.6 | reserved_MB: 28586.0 | inactive_split_MB: 303.9 +rank7 [2026-10-27 21:11:50,556] INFO : Step 73000/100000 rank7 | Train Loss: 12.7380 | Train Loss Unweighted: 4.2392 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.4842 | Regression_cos_phi: 0.0575 | Regression_energy: 0.3093 | Regression_eta: 0.0643 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0567 | Group Train Loss Unweighted (window mean): physical_pu: 3.6309 | physical_nopu: 4.4270 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3555 | Classification_binary: 3.7368 | Regression_cos_phi: 2.7417 | Regression_energy: 1.4936 | Regression_eta: 0.9140 | Regression_pt: 1.6342 | Regression_sin_phi: 2.3690 | LR: 1.01e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.4474s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0007 | max_forward: 0.0985 | max_loss: 0.0632 | max_backward: 0.1654 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 1.2139 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5637.1 | uss_MB: 4897.0 | child_rss_MB: 8681.3 | allocated_MB: 607.0 | reserved_MB: 28586.0 | inactive_split_MB: 227.5 +rank7 [2026-10-27 21:15:34,268] INFO : Step 74000/100000 rank7 | Train Loss: 12.5735 | Train Loss Unweighted: 4.2478 | Train Loss Components: Classification: 0.0162 | Classification_binary: 3.4076 | Regression_cos_phi: 0.0429 | Regression_energy: 0.3533 | Regression_eta: 0.0699 | Regression_pt: 0.3116 | Regression_sin_phi: 0.0463 | Group Train Loss Unweighted (window mean): physical_pu: 3.6307 | physical_nopu: 4.4218 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5692 | Classification_binary: 3.5582 | Regression_cos_phi: 1.8759 | Regression_energy: 2.0466 | Regression_eta: 1.5257 | Regression_pt: 2.4139 | Regression_sin_phi: 2.2744 | LR: 9.68e-05 | DataLoad Time: 0.0021s | Model Forward Time: 2.5465s | Phase Times: max_pre_step_barrier: 0.1687 | max_batch_to_device: 0.0008 | max_forward: 0.0990 | max_loss: 0.0516 | max_backward: 0.1766 | max_optimizer_step: 0.1812 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5626.2 | uss_MB: 4897.3 | child_rss_MB: 8634.5 | allocated_MB: 608.6 | reserved_MB: 28586.0 | inactive_split_MB: 285.4 +rank7 [2026-10-27 21:19:21,651] INFO : Step 75000/100000 rank7 | Train Loss: 15.1687 | Train Loss Unweighted: 4.6098 | Train Loss Components: Classification: 0.0148 | Classification_binary: 3.3751 | Regression_cos_phi: 0.0492 | Regression_energy: 0.5530 | Regression_eta: 0.0638 | Regression_pt: 0.5060 | Regression_sin_phi: 0.0479 | Group Train Loss Unweighted (window mean): physical_pu: 3.6327 | physical_nopu: 4.4206 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5518 | Classification_binary: 3.3536 | Regression_cos_phi: 2.1380 | Regression_energy: 3.1970 | Regression_eta: 1.2713 | Regression_pt: 3.9099 | Regression_sin_phi: 2.1010 | LR: 9.27e-05 | DataLoad Time: 0.0024s | Model Forward Time: 2.4833s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0009 | max_forward: 0.1026 | max_loss: 0.0037 | max_backward: 0.1031 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.2820 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5619.2 | uss_MB: 4896.7 | child_rss_MB: 8602.7 | allocated_MB: 610.1 | reserved_MB: 28586.0 | inactive_split_MB: 259.7 +rank7 [2026-10-27 21:23:05,981] INFO : Step 76000/100000 rank7 | Train Loss: 12.2943 | Train Loss Unweighted: 4.2773 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5085 | Regression_cos_phi: 0.0476 | Regression_energy: 0.3262 | Regression_eta: 0.0732 | Regression_pt: 0.2631 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6289 | physical_nopu: 4.4209 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3598 | Classification_binary: 3.4032 | Regression_cos_phi: 1.5057 | Regression_energy: 1.2585 | Regression_eta: 0.6018 | Regression_pt: 1.4612 | Regression_sin_phi: 1.9432 | LR: 8.88e-05 | DataLoad Time: 0.0022s | Model Forward Time: 2.6227s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.1085 | max_optimizer_step: 0.1831 | max_tensorboard_logging: 1.4262 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 5617.3 | uss_MB: 4895.6 | child_rss_MB: 8601.6 | allocated_MB: 608.9 | reserved_MB: 28942.0 | inactive_split_MB: 227.1 +rank7 [2026-10-27 21:27:01,963] INFO : Step 77000/100000 rank7 | Train Loss: 13.8477 | Train Loss Unweighted: 4.4496 | Train Loss Components: Classification: 0.0156 | Classification_binary: 3.4225 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4510 | Regression_eta: 0.0623 | Regression_pt: 0.4032 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6274 | physical_nopu: 4.4166 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3498 | Classification_binary: 3.3827 | Regression_cos_phi: 1.9695 | Regression_energy: 1.8057 | Regression_eta: 0.9636 | Regression_pt: 1.9914 | Regression_sin_phi: 2.1333 | LR: 8.50e-05 | DataLoad Time: 0.0020s | Model Forward Time: 6.1303s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0839 | max_forward: 0.1754 | max_loss: 0.0036 | max_backward: 0.1103 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 4.6772 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1374 | Memory: rss_MB: 5607.3 | uss_MB: 4838.5 | child_rss_MB: 8502.5 | allocated_MB: 611.0 | reserved_MB: 28942.0 | inactive_split_MB: 369.0 +rank7 [2026-10-27 21:30:52,663] INFO : Step 78000/100000 rank7 | Train Loss: 14.1981 | Train Loss Unweighted: 4.6212 | Train Loss Components: Classification: 0.0133 | Classification_binary: 3.5747 | Regression_cos_phi: 0.0496 | Regression_energy: 0.4573 | Regression_eta: 0.0669 | Regression_pt: 0.4114 | Regression_sin_phi: 0.0481 | Group Train Loss Unweighted (window mean): physical_pu: 3.6241 | physical_nopu: 4.4230 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3035 | Classification_binary: 3.6692 | Regression_cos_phi: 1.6316 | Regression_energy: 1.7064 | Regression_eta: 0.9134 | Regression_pt: 1.8908 | Regression_sin_phi: 1.6401 | LR: 8.13e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.0989s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0009 | max_forward: 0.1149 | max_loss: 0.0636 | max_backward: 0.1914 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 1.8167 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5607.3 | uss_MB: 4897.1 | child_rss_MB: 8495.1 | allocated_MB: 606.6 | reserved_MB: 28942.0 | inactive_split_MB: 309.2 +rank7 [2026-10-27 21:34:49,720] INFO : Step 79000/100000 rank7 | Train Loss: 12.3648 | Train Loss Unweighted: 4.2396 | Train Loss Components: Classification: 0.0145 | Classification_binary: 3.4636 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3226 | Regression_eta: 0.0628 | Regression_pt: 0.2792 | Regression_sin_phi: 0.0491 | Group Train Loss Unweighted (window mean): physical_pu: 3.6254 | physical_nopu: 4.4183 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4807 | Classification_binary: 3.2660 | Regression_cos_phi: 1.7560 | Regression_energy: 1.8732 | Regression_eta: 0.6606 | Regression_pt: 2.2325 | Regression_sin_phi: 1.9065 | LR: 7.78e-05 | DataLoad Time: 0.0016s | Model Forward Time: 3.3391s | Phase Times: max_pre_step_barrier: 0.0008 | max_batch_to_device: 0.0936 | max_forward: 0.1936 | max_loss: 0.0037 | max_backward: 0.1134 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 2.0079 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 5602.4 | uss_MB: 4897.2 | child_rss_MB: 8483.1 | allocated_MB: 609.2 | reserved_MB: 29222.0 | inactive_split_MB: 324.6 +rank7 [2026-10-27 21:38:37,320] INFO : Step 80000/100000 rank7 | Train Loss: 16.1833 | Train Loss Unweighted: 4.8046 | Train Loss Components: Classification: 0.0155 | Classification_binary: 3.4149 | Regression_cos_phi: 0.0453 | Regression_energy: 0.6189 | Regression_eta: 0.0801 | Regression_pt: 0.5835 | Regression_sin_phi: 0.0465 | Group Train Loss Unweighted (window mean): physical_pu: 3.6233 | physical_nopu: 4.4142 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4411 | Classification_binary: 3.2968 | Regression_cos_phi: 1.9332 | Regression_energy: 3.4367 | Regression_eta: 1.0579 | Regression_pt: 4.3073 | Regression_sin_phi: 1.6229 | LR: 7.44e-05 | DataLoad Time: 0.0108s | Model Forward Time: 3.9230s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0008 | max_forward: 0.1071 | max_loss: 0.0037 | max_backward: 0.1380 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 2.6784 | max_tensorboard_flush: 0.0026 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 5596.4 | uss_MB: 4871.2 | child_rss_MB: 8436.2 | allocated_MB: 606.3 | reserved_MB: 29222.0 | inactive_split_MB: 471.7 +rank7 [2026-10-27 21:38:38,322] INFO : Running validation on rank7 +rank7 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 21:38:38,322] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 21:40:39,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 21:40:39,981] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 21:41:05,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 21:41:06,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 21:41:06,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 21:41:07,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 21:41:07,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 21:41:08,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 21:41:08,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 21:41:09,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 21:41:19,336] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 21:41:19,914] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 21:41:20,296] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 21:41:35,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 21:41:35,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 21:41:36,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 21:41:37,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 21:41:37,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 21:41:38,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 21:41:38,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 21:41:39,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 21:41:49,124] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-27 21:41:49,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 21:41:50,005] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 21:42:05,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 21:42:05,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 21:42:06,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 21:42:06,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 21:42:07,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 21:42:07,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 21:42:08,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 21:42:08,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_80000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 21:42:18,588] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 21:46:50,145] INFO : Step 81000/100000 rank7 | Train Loss: 11.7500 | Train Loss Unweighted: 3.6065 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.8934 | Regression_cos_phi: 0.0471 | Regression_energy: 0.2712 | Regression_eta: 0.1074 | Regression_pt: 0.2244 | Regression_sin_phi: 0.0471 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4130 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5056 | Classification_binary: 2.9086 | Regression_cos_phi: 1.8525 | Regression_energy: 1.4806 | Regression_eta: 1.6311 | Regression_pt: 1.6614 | Regression_sin_phi: 1.9362 | LR: 7.11e-05 | DataLoad Time: 0.0020s | Model Forward Time: 8.6906s | Phase Times: max_pre_step_barrier: 0.0010 | max_batch_to_device: 0.0975 | max_forward: 0.3237 | max_loss: 0.0036 | max_backward: 0.5496 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 6.3197 | max_tensorboard_flush: 0.0011 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6963.5 | uss_MB: 4835.8 | child_rss_MB: 8415.8 | allocated_MB: 619.9 | reserved_MB: 28598.0 | inactive_split_MB: 1192.4 +rank7 [2026-10-27 21:50:33,876] INFO : Step 82000/100000 rank7 | Train Loss: 13.6704 | Train Loss Unweighted: 4.5604 | Train Loss Components: Classification: 0.0157 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0419 | Regression_energy: 0.4618 | Regression_eta: 0.0599 | Regression_pt: 0.4291 | Regression_sin_phi: 0.0407 | Group Train Loss Unweighted (window mean): physical_pu: 3.6250 | physical_nopu: 4.4152 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4019 | Classification_binary: 3.5472 | Regression_cos_phi: 1.7984 | Regression_energy: 1.7705 | Regression_eta: 1.2239 | Regression_pt: 2.1396 | Regression_sin_phi: 1.5869 | LR: 6.80e-05 | DataLoad Time: 0.0022s | Model Forward Time: 3.1348s | Phase Times: max_pre_step_barrier: 0.1835 | max_batch_to_device: 0.0874 | max_forward: 0.1976 | max_loss: 0.0036 | max_backward: 0.1170 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 1.6260 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 5659.6 | uss_MB: 4976.9 | child_rss_MB: 8356.3 | allocated_MB: 607.5 | reserved_MB: 28598.0 | inactive_split_MB: 253.8 +rank7 [2026-10-27 21:54:19,227] INFO : Step 83000/100000 rank7 | Train Loss: 12.0270 | Train Loss Unweighted: 4.2414 | Train Loss Components: Classification: 0.0092 | Classification_binary: 3.5113 | Regression_cos_phi: 0.0499 | Regression_energy: 0.3056 | Regression_eta: 0.0671 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0472 | Group Train Loss Unweighted (window mean): physical_pu: 3.6253 | physical_nopu: 4.4141 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.2745 | Classification_binary: 3.5128 | Regression_cos_phi: 2.0831 | Regression_energy: 1.5854 | Regression_eta: 1.1430 | Regression_pt: 1.5780 | Regression_sin_phi: 2.2073 | LR: 6.51e-05 | DataLoad Time: 0.0013s | Model Forward Time: 3.7073s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0008 | max_forward: 0.1966 | max_loss: 0.0138 | max_backward: 0.1124 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 2.4026 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 5659.6 | uss_MB: 4977.1 | child_rss_MB: 8350.2 | allocated_MB: 609.6 | reserved_MB: 28598.0 | inactive_split_MB: 289.5 +rank7 [2026-10-27 21:58:08,516] INFO : Step 84000/100000 rank7 | Train Loss: 12.5172 | Train Loss Unweighted: 4.2412 | Train Loss Components: Classification: 0.0176 | Classification_binary: 3.4173 | Regression_cos_phi: 0.0451 | Regression_energy: 0.3464 | Regression_eta: 0.0637 | Regression_pt: 0.3052 | Regression_sin_phi: 0.0460 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4149 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5333 | Classification_binary: 3.4734 | Regression_cos_phi: 1.6360 | Regression_energy: 1.7956 | Regression_eta: 0.9652 | Regression_pt: 1.9839 | Regression_sin_phi: 1.9233 | LR: 6.23e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.2647s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0008 | max_forward: 0.1103 | max_loss: 0.0037 | max_backward: 0.1159 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 2.0319 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5655.6 | uss_MB: 4977.8 | child_rss_MB: 8312.0 | allocated_MB: 605.9 | reserved_MB: 28598.0 | inactive_split_MB: 208.8 +rank7 [2026-10-27 22:01:46,561] INFO : Step 85000/100000 rank7 | Train Loss: 15.2814 | Train Loss Unweighted: 4.6863 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.3811 | Regression_cos_phi: 0.0432 | Regression_energy: 0.5842 | Regression_eta: 0.0608 | Regression_pt: 0.5579 | Regression_sin_phi: 0.0423 | Group Train Loss Unweighted (window mean): physical_pu: 3.6173 | physical_nopu: 4.4078 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5071 | Classification_binary: 3.4539 | Regression_cos_phi: 1.5502 | Regression_energy: 2.9727 | Regression_eta: 0.9697 | Regression_pt: 3.8520 | Regression_sin_phi: 1.6989 | LR: 5.96e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.0835s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.0993 | max_loss: 0.0558 | max_backward: 0.1762 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 1.8022 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 5645.3 | uss_MB: 4977.4 | child_rss_MB: 8242.1 | allocated_MB: 606.6 | reserved_MB: 28598.0 | inactive_split_MB: 257.4 +rank7 [2026-10-27 22:05:28,666] INFO : Step 86000/100000 rank7 | Train Loss: 12.8074 | Train Loss Unweighted: 4.3373 | Train Loss Components: Classification: 0.0111 | Classification_binary: 3.5785 | Regression_cos_phi: 0.0543 | Regression_energy: 0.3116 | Regression_eta: 0.0747 | Regression_pt: 0.2515 | Regression_sin_phi: 0.0556 | Group Train Loss Unweighted (window mean): physical_pu: 3.6255 | physical_nopu: 4.4086 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3236 | Classification_binary: 3.5382 | Regression_cos_phi: 1.6446 | Regression_energy: 1.6451 | Regression_eta: 1.3653 | Regression_pt: 1.8395 | Regression_sin_phi: 1.9347 | LR: 5.71e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.0435s | Phase Times: max_pre_step_barrier: 0.0007 | max_batch_to_device: 0.0008 | max_forward: 0.1215 | max_loss: 0.0037 | max_backward: 0.1229 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 1.6761 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 5636.3 | uss_MB: 4977.9 | child_rss_MB: 8180.8 | allocated_MB: 608.3 | reserved_MB: 29688.0 | inactive_split_MB: 271.8 +rank7 [2026-10-27 22:09:22,620] INFO : Step 87000/100000 rank7 | Train Loss: 12.9093 | Train Loss Unweighted: 4.3003 | Train Loss Components: Classification: 0.0172 | Classification_binary: 3.3807 | Regression_cos_phi: 0.0447 | Regression_energy: 0.4014 | Regression_eta: 0.0615 | Regression_pt: 0.3523 | Regression_sin_phi: 0.0424 | Group Train Loss Unweighted (window mean): physical_pu: 3.6211 | physical_nopu: 4.4120 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4870 | Classification_binary: 3.2765 | Regression_cos_phi: 1.8360 | Regression_energy: 3.3435 | Regression_eta: 1.2498 | Regression_pt: 3.9901 | Regression_sin_phi: 1.7305 | LR: 5.48e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.0852s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0203 | max_forward: 0.1299 | max_loss: 0.0038 | max_backward: 0.1175 | max_optimizer_step: 0.2192 | max_tensorboard_logging: 1.8010 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5636.1 | uss_MB: 4575.9 | child_rss_MB: 8182.7 | allocated_MB: 607.0 | reserved_MB: 29688.0 | inactive_split_MB: 231.0 +rank7 [2026-10-27 22:13:02,799] INFO : Step 88000/100000 rank7 | Train Loss: 15.2369 | Train Loss Unweighted: 4.6456 | Train Loss Components: Classification: 0.0147 | Classification_binary: 3.4525 | Regression_cos_phi: 0.0526 | Regression_energy: 0.5240 | Regression_eta: 0.0665 | Regression_pt: 0.4840 | Regression_sin_phi: 0.0514 | Group Train Loss Unweighted (window mean): physical_pu: 3.6192 | physical_nopu: 4.4119 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3847 | Classification_binary: 3.6748 | Regression_cos_phi: 2.4974 | Regression_energy: 2.3461 | Regression_eta: 1.2725 | Regression_pt: 2.6618 | Regression_sin_phi: 2.2215 | LR: 5.26e-05 | DataLoad Time: 0.0020s | Model Forward Time: 3.1170s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0007 | max_forward: 0.1127 | max_loss: 0.0236 | max_backward: 0.1542 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 1.7956 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5636.1 | uss_MB: 4977.3 | child_rss_MB: 8181.7 | allocated_MB: 609.0 | reserved_MB: 29688.0 | inactive_split_MB: 221.0 +rank7 [2026-10-27 22:16:52,919] INFO : Step 89000/100000 rank7 | Train Loss: 12.8609 | Train Loss Unweighted: 4.2765 | Train Loss Components: Classification: 0.0136 | Classification_binary: 3.4665 | Regression_cos_phi: 0.0490 | Regression_energy: 0.3343 | Regression_eta: 0.0681 | Regression_pt: 0.2896 | Regression_sin_phi: 0.0555 | Group Train Loss Unweighted (window mean): physical_pu: 3.6177 | physical_nopu: 4.4110 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4142 | Classification_binary: 3.5410 | Regression_cos_phi: 1.6329 | Regression_energy: 1.5202 | Regression_eta: 0.9774 | Regression_pt: 1.7787 | Regression_sin_phi: 1.6296 | LR: 5.06e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.1956s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0009 | max_forward: 0.1194 | max_loss: 0.0037 | max_backward: 0.1299 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 1.9452 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 5632.5 | uss_MB: 4977.7 | child_rss_MB: 8137.7 | allocated_MB: 612.6 | reserved_MB: 30628.0 | inactive_split_MB: 309.7 +rank7 [2026-10-27 22:20:43,711] INFO : Step 90000/100000 rank7 | Train Loss: 11.6102 | Train Loss Unweighted: 3.6127 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.9177 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2661 | Regression_eta: 0.1040 | Regression_pt: 0.2155 | Regression_sin_phi: 0.0470 | Group Train Loss Unweighted (window mean): physical_pu: 3.6210 | physical_nopu: 4.4109 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4524 | Classification_binary: 2.9516 | Regression_cos_phi: 1.8324 | Regression_energy: 1.4298 | Regression_eta: 1.5523 | Regression_pt: 1.5519 | Regression_sin_phi: 1.8822 | LR: 4.88e-05 | DataLoad Time: 0.0023s | Model Forward Time: 3.5770s | Phase Times: max_pre_step_barrier: 0.0009 | max_batch_to_device: 0.0110 | max_forward: 0.1856 | max_loss: 0.0145 | max_backward: 0.4526 | max_optimizer_step: 0.1917 | max_tensorboard_logging: 1.8620 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 5624.5 | uss_MB: 4985.7 | child_rss_MB: 8062.4 | allocated_MB: 617.2 | reserved_MB: 30628.0 | inactive_split_MB: 962.2 +rank7 [2026-10-27 22:20:44,690] INFO : Running validation on rank7 +rank7 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 22:20:44,690] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 22:22:45,205] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 22:22:45,606] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 22:23:10,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 22:23:10,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 22:23:11,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 22:23:12,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 22:23:12,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 22:23:13,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 22:23:13,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 22:23:14,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 22:23:24,060] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 22:23:24,365] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 22:23:24,734] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 22:23:40,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 22:23:40,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 22:23:41,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 22:23:41,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 22:23:42,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 22:23:42,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 22:23:43,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 22:23:43,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 22:23:53,900] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 22:23:53,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 22:23:54,276] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 22:24:09,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 22:24:09,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 22:24:10,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 22:24:10,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 22:24:11,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 22:24:11,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 22:24:12,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 22:24:12,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_90000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 22:24:22,920] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 22:28:41,211] INFO : Step 91000/100000 rank7 | Train Loss: 12.9063 | Train Loss Unweighted: 4.3371 | Train Loss Components: Classification: 0.0125 | Classification_binary: 3.5236 | Regression_cos_phi: 0.0531 | Regression_energy: 0.3431 | Regression_eta: 0.0707 | Regression_pt: 0.2829 | Regression_sin_phi: 0.0512 | Group Train Loss Unweighted (window mean): physical_pu: 3.6180 | physical_nopu: 4.4087 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3967 | Classification_binary: 3.4422 | Regression_cos_phi: 2.0624 | Regression_energy: 2.6762 | Regression_eta: 1.2638 | Regression_pt: 2.7916 | Regression_sin_phi: 3.0877 | LR: 4.71e-05 | DataLoad Time: 0.0014s | Model Forward Time: 4.9004s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0055 | max_forward: 0.1521 | max_loss: 0.0037 | max_backward: 0.1649 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 3.4850 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7066.6 | uss_MB: 6432.9 | child_rss_MB: 8020.4 | allocated_MB: 607.8 | reserved_MB: 28796.0 | inactive_split_MB: 320.2 +rank7 [2026-10-27 22:32:23,431] INFO : Step 92000/100000 rank7 | Train Loss: 15.4341 | Train Loss Unweighted: 4.7247 | Train Loss Components: Classification: 0.0160 | Classification_binary: 3.4285 | Regression_cos_phi: 0.0457 | Regression_energy: 0.5811 | Regression_eta: 0.0735 | Regression_pt: 0.5375 | Regression_sin_phi: 0.0425 | Group Train Loss Unweighted (window mean): physical_pu: 3.6150 | physical_nopu: 4.4032 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4601 | Classification_binary: 3.3421 | Regression_cos_phi: 1.7604 | Regression_energy: 2.4981 | Regression_eta: 0.9993 | Regression_pt: 3.0063 | Regression_sin_phi: 1.7712 | LR: 4.57e-05 | DataLoad Time: 0.0018s | Model Forward Time: 3.3949s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0009 | max_forward: 0.2696 | max_loss: 0.0037 | max_backward: 0.1715 | max_optimizer_step: 0.3237 | max_tensorboard_logging: 1.7768 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7051.6 | uss_MB: 6432.9 | child_rss_MB: 7931.4 | allocated_MB: 609.3 | reserved_MB: 28796.0 | inactive_split_MB: 222.8 +rank7 [2026-10-27 22:36:08,796] INFO : Step 93000/100000 rank7 | Train Loss: 13.9744 | Train Loss Unweighted: 4.4854 | Train Loss Components: Classification: 0.0118 | Classification_binary: 3.5124 | Regression_cos_phi: 0.0578 | Regression_energy: 0.4245 | Regression_eta: 0.0652 | Regression_pt: 0.3624 | Regression_sin_phi: 0.0513 | Group Train Loss Unweighted (window mean): physical_pu: 3.6205 | physical_nopu: 4.4075 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4473 | Classification_binary: 3.4710 | Regression_cos_phi: 1.8002 | Regression_energy: 1.9650 | Regression_eta: 1.1530 | Regression_pt: 2.2188 | Regression_sin_phi: 1.9730 | LR: 4.43e-05 | DataLoad Time: 0.0025s | Model Forward Time: 2.9846s | Phase Times: max_pre_step_barrier: 0.0011 | max_batch_to_device: 0.0008 | max_forward: 0.1098 | max_loss: 0.0036 | max_backward: 0.1173 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 1.7709 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5706.9 | uss_MB: 5092.9 | child_rss_MB: 7894.0 | allocated_MB: 606.4 | reserved_MB: 30394.0 | inactive_split_MB: 244.2 +rank7 [2026-10-27 22:39:53,036] INFO : Step 94000/100000 rank7 | Train Loss: 12.4397 | Train Loss Unweighted: 4.2538 | Train Loss Components: Classification: 0.0159 | Classification_binary: 3.4230 | Regression_cos_phi: 0.0415 | Regression_energy: 0.3513 | Regression_eta: 0.0702 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0452 | Group Train Loss Unweighted (window mean): physical_pu: 3.6164 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5461 | Classification_binary: 3.3727 | Regression_cos_phi: 1.3872 | Regression_energy: 2.5436 | Regression_eta: 0.7906 | Regression_pt: 3.1095 | Regression_sin_phi: 1.5492 | LR: 4.32e-05 | DataLoad Time: 0.0019s | Model Forward Time: 3.1514s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0009 | max_forward: 0.1540 | max_loss: 0.0036 | max_backward: 0.2309 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 1.7723 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 5695.9 | uss_MB: 4753.0 | child_rss_MB: 7806.9 | allocated_MB: 607.1 | reserved_MB: 30394.0 | inactive_split_MB: 232.2 +rank7 [2026-10-27 22:43:36,954] INFO : Step 95000/100000 rank7 | Train Loss: 14.9177 | Train Loss Unweighted: 4.6108 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.3911 | Regression_cos_phi: 0.0436 | Regression_energy: 0.5612 | Regression_eta: 0.0598 | Regression_pt: 0.4905 | Regression_sin_phi: 0.0493 | Group Train Loss Unweighted (window mean): physical_pu: 3.6121 | physical_nopu: 4.4076 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3851 | Classification_binary: 3.5128 | Regression_cos_phi: 1.6738 | Regression_energy: 2.1924 | Regression_eta: 1.0926 | Regression_pt: 2.5903 | Regression_sin_phi: 1.9908 | LR: 4.22e-05 | DataLoad Time: 0.0010s | Model Forward Time: 3.0362s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0008 | max_forward: 0.1111 | max_loss: 0.0199 | max_backward: 0.1395 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.7978 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5695.9 | uss_MB: 5093.1 | child_rss_MB: 7811.5 | allocated_MB: 609.0 | reserved_MB: 30394.0 | inactive_split_MB: 285.3 +rank7 [2026-10-27 22:47:14,355] INFO : Step 96000/100000 rank7 | Train Loss: 12.4738 | Train Loss Unweighted: 4.2244 | Train Loss Components: Classification: 0.0098 | Classification_binary: 3.4884 | Regression_cos_phi: 0.0540 | Regression_energy: 0.2967 | Regression_eta: 0.0804 | Regression_pt: 0.2429 | Regression_sin_phi: 0.0522 | Group Train Loss Unweighted (window mean): physical_pu: 3.6137 | physical_nopu: 4.4081 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3358 | Classification_binary: 3.5400 | Regression_cos_phi: 1.9727 | Regression_energy: 1.6360 | Regression_eta: 1.6135 | Regression_pt: 1.7422 | Regression_sin_phi: 2.4029 | LR: 4.14e-05 | DataLoad Time: 0.0026s | Model Forward Time: 3.1603s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0007 | max_forward: 0.1063 | max_loss: 0.0036 | max_backward: 0.2003 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 1.8675 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5695.9 | uss_MB: 5093.2 | child_rss_MB: 7804.9 | allocated_MB: 608.1 | reserved_MB: 30394.0 | inactive_split_MB: 259.4 +rank7 [2026-10-27 22:50:52,983] INFO : Step 97000/100000 rank7 | Train Loss: 12.7922 | Train Loss Unweighted: 4.2852 | Train Loss Components: Classification: 0.0170 | Classification_binary: 3.4355 | Regression_cos_phi: 0.0455 | Regression_energy: 0.3604 | Regression_eta: 0.0703 | Regression_pt: 0.3091 | Regression_sin_phi: 0.0474 | Group Train Loss Unweighted (window mean): physical_pu: 3.6136 | physical_nopu: 4.4015 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.5958 | Classification_binary: 3.4209 | Regression_cos_phi: 1.5823 | Regression_energy: 1.5158 | Regression_eta: 0.9200 | Regression_pt: 1.8507 | Regression_sin_phi: 1.7837 | LR: 4.08e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.0105s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0008 | max_forward: 0.1110 | max_loss: 0.0174 | max_backward: 0.1239 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 1.7282 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0908 | Memory: rss_MB: 5687.4 | uss_MB: 5092.4 | child_rss_MB: 7746.9 | allocated_MB: 607.2 | reserved_MB: 30394.0 | inactive_split_MB: 265.3 +rank7 [2026-10-27 22:54:49,599] INFO : Step 98000/100000 rank7 | Train Loss: 13.8508 | Train Loss Unweighted: 4.4512 | Train Loss Components: Classification: 0.0149 | Classification_binary: 3.3736 | Regression_cos_phi: 0.0443 | Regression_energy: 0.4792 | Regression_eta: 0.0606 | Regression_pt: 0.4354 | Regression_sin_phi: 0.0433 | Group Train Loss Unweighted (window mean): physical_pu: 3.6169 | physical_nopu: 4.4033 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4462 | Classification_binary: 3.4798 | Regression_cos_phi: 1.5780 | Regression_energy: 3.0420 | Regression_eta: 0.8796 | Regression_pt: 3.8538 | Regression_sin_phi: 2.0349 | LR: 4.04e-05 | DataLoad Time: 0.0024s | Model Forward Time: 3.0654s | Phase Times: max_pre_step_barrier: 0.0012 | max_batch_to_device: 0.0119 | max_forward: 0.1317 | max_loss: 0.0036 | max_backward: 0.1246 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.7312 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0878 | Memory: rss_MB: 5687.4 | uss_MB: 5093.6 | child_rss_MB: 7731.7 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 292.7 +rank7 [2026-10-27 22:58:30,099] INFO : Step 99000/100000 rank7 | Train Loss: 12.5729 | Train Loss Unweighted: 4.3128 | Train Loss Components: Classification: 0.0124 | Classification_binary: 3.5536 | Regression_cos_phi: 0.0519 | Regression_energy: 0.3097 | Regression_eta: 0.0644 | Regression_pt: 0.2679 | Regression_sin_phi: 0.0529 | Group Train Loss Unweighted (window mean): physical_pu: 3.6154 | physical_nopu: 4.4035 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.3511 | Classification_binary: 3.5490 | Regression_cos_phi: 1.8988 | Regression_energy: 1.3887 | Regression_eta: 0.8995 | Regression_pt: 1.4832 | Regression_sin_phi: 1.9744 | LR: 4.01e-05 | DataLoad Time: 0.0113s | Model Forward Time: 3.0443s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0008 | max_forward: 0.1158 | max_loss: 0.0113 | max_backward: 0.1168 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 1.7726 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 5687.4 | uss_MB: 5093.5 | child_rss_MB: 7727.1 | allocated_MB: 606.3 | reserved_MB: 30394.0 | inactive_split_MB: 283.7 +rank7 [2026-10-27 23:02:17,099] INFO : Step 100000/100000 rank7 | Train Loss: 11.9488 | Train Loss Unweighted: 3.6450 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.9057 | Regression_cos_phi: 0.0475 | Regression_energy: 0.2836 | Regression_eta: 0.1038 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0484 | Group Train Loss Unweighted (window mean): physical_pu: 3.6201 | physical_nopu: 4.4070 | Task Weights: Classification: 30.9399 | Classification_binary: 1.0000 | Regression_cos_phi: 37.4041 | Regression_energy: 5.3692 | Regression_eta: 15.3039 | Regression_pt: 7.1565 | Regression_sin_phi: 40.2399 | Task Weighted Losses: Classification: 0.4689 | Classification_binary: 2.8868 | Regression_cos_phi: 1.7279 | Regression_energy: 1.5792 | Regression_eta: 1.6254 | Regression_pt: 1.8183 | Regression_sin_phi: 1.9657 | LR: 4.00e-05 | DataLoad Time: 0.0031s | Model Forward Time: 6.3805s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0031 | max_forward: 0.2860 | max_loss: 0.0036 | max_backward: 0.4897 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 4.4601 | max_tensorboard_flush: 0.0009 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5687.2 | uss_MB: 4556.1 | child_rss_MB: 7711.3 | allocated_MB: 618.6 | reserved_MB: 30394.0 | inactive_split_MB: 1112.1 +rank7 [2026-10-27 23:02:18,080] INFO : Running validation on rank7 +rank7 [2026-10-27 23:02:18,080] INFO : ResumableSampler physical_pu:valid resetting +rank7 [2026-10-27 23:02:18,081] INFO : ResumableSampler physical_pu:valid loading state: start_index=0 +rank7 [2026-10-27 23:02:18,081] INFO : ResumableSampler physical_nopu:valid resetting +rank7 [2026-10-27 23:02:18,081] INFO : ResumableSampler physical_nopu:valid loading state: start_index=0 +rank7 [2026-10-27 23:04:23,625] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 23:04:24,087] INFO : Running predictions on cms_pf_ttbar +rank7 [2026-10-27 23:04:49,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_7_0.parquet +rank7 [2026-10-27 23:04:50,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_7_1.parquet +rank7 [2026-10-27 23:04:50,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_7_2.parquet +rank7 [2026-10-27 23:04:51,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_7_3.parquet +rank7 [2026-10-27 23:04:51,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_7_4.parquet +rank7 [2026-10-27 23:04:52,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_7_5.parquet +rank7 [2026-10-27 23:04:52,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_7_6.parquet +rank7 [2026-10-27 23:04:53,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ttbar/pred_7_7.parquet +rank7 [2026-10-27 23:05:03,227] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-27 23:05:03,812] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 23:05:04,178] INFO : Running predictions on cms_pf_qcd +rank7 [2026-10-27 23:05:19,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_7_0.parquet +rank7 [2026-10-27 23:05:20,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_7_1.parquet +rank7 [2026-10-27 23:05:21,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_7_2.parquet +rank7 [2026-10-27 23:05:21,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_7_3.parquet +rank7 [2026-10-27 23:05:22,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_7_4.parquet +rank7 [2026-10-27 23:05:22,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_7_5.parquet +rank7 [2026-10-27 23:05:23,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_7_6.parquet +rank7 [2026-10-27 23:05:23,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_qcd/pred_7_7.parquet +rank7 [2026-10-27 23:05:33,612] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-27 23:05:33,778] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-27 23:05:34,200] INFO : Running predictions on cms_pf_ztt +rank7 [2026-10-27 23:05:49,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_7_0.parquet +rank7 [2026-10-27 23:05:50,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_7_1.parquet +rank7 [2026-10-27 23:05:50,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_7_2.parquet +rank7 [2026-10-27 23:05:51,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_7_3.parquet +rank7 [2026-10-27 23:05:51,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_7_4.parquet +rank7 [2026-10-27 23:05:52,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_7_5.parquet +rank7 [2026-10-27 23:05:52,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_7_6.parquet +rank7 [2026-10-27 23:05:53,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/cms_backbone_comparison/heptv2_seed12345_20261027_160940_124470/preds_step_100000/cms_pf_ztt/pred_7_7.parquet +rank7 [2026-10-27 23:06:03,326] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-27 23:06:03,958] INFO : Training completed. Total time on device 7: 414.788min diff --git a/cms/v3.2.0/release.json b/cms/v3.2.0/release.json new file mode 100644 index 0000000000000000000000000000000000000000..be371af75d2748e5aa4de548f06ebc2400d78811 --- /dev/null +++ b/cms/v3.2.0/release.json @@ -0,0 +1,73 @@ +{ + "release": "3.2.0", + "dataset_version": "2.8.0", + "selected_step": 100000, + "models": { + "attention": { + "run": "attention_seed12345_20261027_160940_955225", + "checkpoint": "checkpoints/checkpoint-100000.pth", + "checkpoint_sha256": "96c8e1aa40a53f4b33cae9dafddd169a555c15fc75f987a9fdf13a61b46e4934", + "parameters": 12421153, + "validation_metrics": { + "Total_unweighted": 4.0335239842534065, + "metrics/particle/matching/f1": 0.6246592998504639, + "Regression_pt": 0.30063915252685547, + "metrics/particle/event/met_abs_error": 18.390907287597656 + }, + "sampler_state": { + "batches_in_epoch": [ + 9090, + 90910 + ], + "epochs": [ + 0, + 0 + ], + "scheduled": [ + 9090, + 90910 + ], + "weights": [ + 21093.0, + 210965.0 + ] + } + }, + "heptv2": { + "run": "heptv2_seed12345_20261027_160940_124470", + "checkpoint": "checkpoints/checkpoint-100000.pth", + "checkpoint_sha256": "8ffec8bfb5c8b409282c32bcfd1f4bd53b47de21ad500e878273150436af9da8", + "parameters": 23450849, + "validation_metrics": { + "Total_unweighted": 4.012098126113415, + "metrics/particle/matching/f1": 0.6279647350311279, + "Regression_pt": 0.2949534257253011, + "metrics/particle/event/met_abs_error": 18.393125534057617 + }, + "sampler_state": { + "batches_in_epoch": [ + 9090, + 90910 + ], + "epochs": [ + 0, + 0 + ], + "scheduled": [ + 9090, + 90910 + ], + "weights": [ + 21093.0, + 210965.0 + ] + } + } + }, + "contents": "Final 100k checkpoints, complete histories, train/valid TensorBoard streams, configs, model kwargs, manifests, all training logs, and final plots. Prediction parquet files and batch dumps are excluded.", + "comparison_metrics": { + "description": "Reference checkpoint evaluated in FP32 on the same saved PU test inputs as the new BF16 predictions: 512 entries per QCD, ttbar, Ztt, 1530 unique events. Separate from mixed PU/noPU validation metrics in histories.", + "reference": "https://huggingface.co/jpata/particleflow/tree/main/cms/v2.6.0pre1/pyg-cms_20250722_101813_274478", + "code_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc" + } +}